yaralyzer 1.0.11__py3-none-any.whl

yaralyzer/yara/yara_rule_builder.py

@@ -0,0 +1,164 @@
+"""
+Builds bare bones YARA rules to match strings and regex patterns.
+
+Example rule string:
+
+```
+rule Just_A_Piano_Man {
+    meta:
+        author           = "Tim"
+    strings:
+        $hilton_producer = /Scott.*Storch/
+    condition:
+        $hilton_producer
+}
+```
+"""
+import re
+from typing import Literal, Optional
+
+import yara
+
+from yaralyzer.config import YARALYZE
+from yaralyzer.util.logging import log
+
+PatternType = Literal['hex', 'regex']
+YaraModifierType = Literal['ascii', 'fullword', 'nocase', 'wide']
+
+HEX = 'hex'
+PATTERN = 'pattern'
+REGEX = 'regex'
+RULE = 'rule'
+UNDERSCORE = '_'
+YARA_REGEX_MODIFIERS = ['nocase', 'ascii', 'wide', 'fullword']
+
+SAFE_LABEL_REPLACEMENTS = {
+    '/': 'frontslash',
+    '\\': 'backslash',
+    "'": 'singlequote',
+    '"': 'doublequote',
+    '`': 'backtick',
+    '-': UNDERSCORE,
+    ' ': UNDERSCORE,
+}
+
+RULE_TEMPLATE = """
+rule {rule_name} {{
+    meta:
+        author = "The Yaralyzer"
+    strings:
+        ${pattern_label} = {pattern}
+    condition:
+        ${pattern_label}
+}}
+"""
+
+BYTES_RULE_TEMPLATE = """
+rule {rule_name} {{
+    meta:
+        author = "The Yaralyzer"
+    strings:
+        ${pattern_label} = {{ {bytes_pattern} }}
+    condition:
+        ${pattern_label}
+}}
+"""
+
+
+def yara_rule_string(
+    pattern: str,
+    pattern_type: PatternType = REGEX,
+    rule_name: str = YARALYZE,
+    pattern_label: Optional[str] = PATTERN,
+    modifier: Optional[YaraModifierType] = None
+) -> str:
+    """
+    Build a YARA rule string for a given `pattern`.
+
+    Args:
+        pattern (str): The string or regex pattern to match.
+        pattern_type (str): Either `"regex"` or `"hex"`. Default is `"regex"`.
+        rule_name (str): The name of the YARA rule. Default is `"YARALYZE"`.
+        pattern_label (Optional[str]): The label for the pattern in the YARA rule. Default is `"pattern"`.
+        modifier (Optional[str]): Optional regex modifier (e.g. 'nocase', 'ascii', 'wide', 'fullword').
+            Only valid if `pattern_type` is `"regex"`.
+
+    Returns:
+        str: The constructed YARA rule as a string.
+    """
+    if not (modifier is None or modifier in YARA_REGEX_MODIFIERS):
+        raise TypeError(f"Modifier '{modifier}' is not one of {YARA_REGEX_MODIFIERS}")
+
+    if pattern_type == REGEX:
+        pattern = f"/{pattern}/"
+    elif pattern_type == HEX:
+        pattern = f"{{{pattern}}}"
+    else:
+        raise ValueError(f"pattern_type must be either '{REGEX}' or '{HEX}'")
+
+    if modifier:
+        pattern += f" {modifier}"
+
+    rule = RULE_TEMPLATE.format(
+        rule_name=rule_name,
+        pattern_label=pattern_label,
+        pattern=pattern,
+        modifier='' if modifier is None else f" {modifier}"
+    )
+
+    log.debug(f"Built YARA rule: \n{rule}")
+    return rule
+
+
+def build_yara_rule(
+    pattern: str,
+    pattern_type: PatternType = REGEX,
+    rule_name: str = YARALYZE,
+    pattern_label: Optional[str] = PATTERN,
+    modifier: Optional[YaraModifierType] = None
+) -> yara.Rule:
+    """
+    Build a compiled `yara.Rule` object.
+
+    Args:
+        pattern (str): The string or regex pattern to match.
+        pattern_type (str): Either `"regex"` or `"hex"`. Default is `"regex"`.
+        rule_name (str): The name of the YARA rule. Default is `"YARALYZE"`.
+        pattern_label (Optional[str]): The label for the pattern in the YARA rule. Default is `"pattern"`.
+        modifier (Optional[str]): Optional regex modifier (e.g. 'nocase', 'ascii', 'wide', 'fullword').
+            Only valid if `pattern_type` is `"regex"`.
+
+    Returns:
+        yara.Rule: Compiled YARA rule object.
+    """
+    rule_string = yara_rule_string(pattern, pattern_type, rule_name, pattern_label, modifier)
+    return yara.compile(source=rule_string)
+
+
+def safe_label(_label: str) -> str:
+    """
+    YARA rule and pattern names can only contain alphanumeric chars.
+
+    Args:
+        _label (str): The label to sanitize.
+
+    Returns:
+        str: A sanitized label safe for use in YARA rules.
+    """
+    label = _label
+
+    for char, replacement in SAFE_LABEL_REPLACEMENTS.items():
+        if replacement != UNDERSCORE:
+            label = label.replace(char, f"__{replacement.upper()}__")
+        else:
+            label = label.replace(char, replacement)
+
+    if re.match('^\\d', label):
+        label = '_' + label
+
+    if not re.match('\\w+', label):
+        msg = f"'{label}' is invalid: YARA labels must be alphanumeric/underscore and cannot start with a number"
+        raise ValueError(msg)
+
+    log.debug(f"Built safe label {label} from {_label}")
+    return label

yaralyzer/yaralyzer.py

@@ -0,0 +1,304 @@
+"""Main Yaralyzer class and alternate constructors."""
+from os import path
+from typing import Callable, Iterator, List, Optional, Tuple, Union
+
+import yara
+from rich.console import Console, ConsoleOptions, RenderResult
+from rich.padding import Padding
+from rich.style import Style
+from rich.text import Text
+
+from yaralyzer.bytes_match import BytesMatch
+from yaralyzer.config import YARALYZE, YaralyzerConfig
+from yaralyzer.decoding.bytes_decoder import BytesDecoder
+from yaralyzer.helpers.file_helper import files_in_dir, load_binary_data
+from yaralyzer.helpers.rich_text_helper import dim_if, reverse_color, print_fatal_error_and_exit
+from yaralyzer.helpers.string_helper import comma_join, newline_join
+from yaralyzer.output.file_hashes_table import bytes_hashes_table
+from yaralyzer.output.regex_match_metrics import RegexMatchMetrics
+from yaralyzer.output.rich_console import YARALYZER_THEME, console
+from yaralyzer.util.logging import log
+from yaralyzer.yara.yara_match import YaraMatch
+from yaralyzer.yara.yara_rule_builder import yara_rule_string
+
+YARA_FILE_DOES_NOT_EXIST_ERROR_MSG = "is not a valid yara rules file (it doesn't exist)"
+
+
+# TODO: might be worth introducing a Scannable namedtuple or similar
+class Yaralyzer:
+    """
+    Central class that handles setting up / compiling YARA rules and reading binary data from files as needed.
+
+    Alternate constructors are provided depending on whether:
+
+    * YARA rules are already compiled
+
+    * YARA rules should be compiled from a string
+
+    * YARA rules should be read from a file
+
+    * YARA rules should be read from a directory of .yara files
+
+    The real action happens in the `__rich__console__()` dunder method.
+
+    Attributes:
+        bytes (bytes): The binary data to scan.
+        bytes_length (int): The length of the binary data.
+        scannable_label (str): A label for the binary data, typically the filename or a user-provided label.
+        rules (yara.Rules): The compiled YARA rules to use for scanning.
+        rules_label (str): A label for the ruleset, typically derived from filenames or user input.
+        highlight_style (str): The style to use for highlighting matches in the output.
+        non_matches (List[dict]): A list of YARA rules that did not match the binary data.
+        matches (List[YaraMatch]): A list of YaraMatch objects representing the matches found.
+        extraction_stats (RegexMatchMetrics): Metrics related to decoding attempts on matched data
+    """
+
+    def __init__(
+        self,
+        rules: Union[str, yara.Rules],
+        rules_label: str,
+        scannable: Union[bytes, str],
+        scannable_label: Optional[str] = None,
+        highlight_style: str = YaralyzerConfig.HIGHLIGHT_STYLE
+    ) -> None:
+        """
+        Initialize a `Yaralyzer` instance for scanning binary data with YARA rules.
+
+        Args:
+            rules (Union[str, yara.Rules]): YARA rules to use for scanning. Can be a string or a pre-compiled
+                `yara.Rules` object (strings will be compiled to an instance of `yara.Rules`).
+            rules_label (str): Label to identify the ruleset in output and logs.
+            scannable (Union[bytes, str]): The data to scan. If it's `bytes` type then that data is scanned;
+                if it's a string it is treated as a file path to load bytes from.
+            scannable_label (Optional[str], optional): Label for the `scannable` arg data.
+                Required if `scannable` is `bytes`.
+                If `scannable` is a file path `scannable_label` will default to the file's basename.
+            highlight_style (str, optional): Style to use for highlighting matches in output.
+                Defaults to `YaralyzerConfig.HIGHLIGHT_STYLE`.
+
+        Raises:
+            TypeError: If `scannable` is `bytes` and `scannable_label` is not provided.
+        """
+        if 'args' not in vars(YaralyzerConfig):
+            YaralyzerConfig.set_default_args()
+
+        yara.set_config(
+            stack_size=YaralyzerConfig.args.yara_stack_size,
+            max_match_data=YaralyzerConfig.args.max_match_length
+        )
+
+        if isinstance(scannable, bytes):
+            if scannable_label is None:
+                raise TypeError("Must provide 'scannable_label' arg when yaralyzing raw bytes")
+
+            self.bytes: bytes = scannable
+            self.scannable_label: str = scannable_label
+        else:
+            self.bytes: bytes = load_binary_data(scannable)
+            self.scannable_label: str = scannable_label or path.basename(scannable)
+
+        if isinstance(rules, yara.Rules):
+            self.rules: yara.Rules = rules
+        else:
+            log.info(f"Compiling YARA rules from provided string:\n{rules}")
+            self.rules: yara.Rules = yara.compile(source=rules)
+
+        self.bytes_length: int = len(self.bytes)
+        self.rules_label: str = rules_label
+        self.highlight_style: str = highlight_style
+        # Outcome tracking variables
+        self.non_matches: List[dict] = []
+        self.matches: List[YaraMatch] = []
+        self.extraction_stats = RegexMatchMetrics()
+
+    @classmethod
+    def for_rules_files(
+        cls,
+        yara_rules_files: List[str],
+        scannable: Union[bytes, str],
+        scannable_label: Optional[str] = None
+    ) -> 'Yaralyzer':
+        """
+        Alternate constructor to load YARA rules from files and label rules with the filenames.
+
+        Args:
+            yara_rules_files (List[str]): List of file paths to YARA rules files.
+            scannable (Union[bytes, str]): The data to scan. If `bytes`, raw data is scanned;
+                if `str`, it is treated as a file path to load bytes from.
+            scannable_label (Optional[str], optional): Label for the `scannable` data.
+                Required if `scannable` is `bytes`. If scannable is a file path, defaults to the file's basename.
+
+        Raises:
+            FileNotFoundError: If any file in `yara_rules_files` does not exist.
+            TypeError: If `yara_rules_files` is not a list.
+        """
+        if not isinstance(yara_rules_files, list):
+            raise TypeError(f"{yara_rules_files} is not a list")
+
+        for file in yara_rules_files:
+            if not path.exists(file):
+                raise FileNotFoundError(f"'{file}' {YARA_FILE_DOES_NOT_EXIST_ERROR_MSG}")
+
+        filepaths_arg = {path.basename(file): file for file in yara_rules_files}
+
+        try:
+            yara_rules = yara.compile(filepaths=filepaths_arg)
+        except yara.SyntaxError as e:
+            print_fatal_error_and_exit(f"Failed to parse YARA rules file(s): {e}")
+
+        yara_rules_label = comma_join(yara_rules_files, func=path.basename)
+        return cls(yara_rules, yara_rules_label, scannable, scannable_label)
+
+    @classmethod
+    def for_rules_dirs(
+        cls,
+        dirs: List[str],
+        scannable: Union[bytes, str],
+        scannable_label: Optional[str] = None
+    ) -> 'Yaralyzer':
+        """
+        Alternate constructor that will load all `.yara` files in `yara_rules_dir`.
+
+        Args:
+            dirs (List[str]): List of directories to search for `.yara` files.
+            scannable (Union[bytes, str]): The data to scan. If `bytes`, raw data is scanned;
+                if `str`, it is treated as a file path to load bytes from.
+            scannable_label (Optional[str], optional): Label for the `scannable` data.
+                Required if `scannable` is `bytes`. If scannable is a file path, defaults to the file's basename.
+
+        Raises:
+            FileNotFoundError: If `dirs` is not a list of valid directories.
+        """
+        if not (isinstance(dirs, list) and all(path.isdir(dir) for dir in dirs)):
+            raise FileNotFoundError(f"'{dirs}' is not a list of valid directories")
+
+        rules_files = [path.join(dir, f) for dir in dirs for f in files_in_dir(dir)]
+        return cls.for_rules_files(rules_files, scannable, scannable_label)
+
+    @classmethod
+    def for_patterns(
+        cls,
+        patterns: List[str],
+        patterns_type: str,
+        scannable: Union[bytes, str],
+        scannable_label: Optional[str] = None,
+        rules_label: Optional[str] = None,
+        pattern_label: Optional[str] = None,
+        regex_modifier: Optional[str] = None,
+    ) -> 'Yaralyzer':
+        """
+        Alternate constructor taking regex pattern strings. Rules label defaults to the patterns joined by comma.
+
+        Args:
+            patterns (List[str]): List of regex or hex patterns to build rules from.
+            patterns_type (str): Either `"regex"` or `"hex"` to indicate the type of patterns provided.
+            scannable (Union[bytes, str]): The data to scan. If `bytes`, raw data is scanned;
+                if `str`, it is treated as a file path to load bytes from.
+            scannable_label (Optional[str], optional): Label for the `scannable` data.
+                Required if `scannable` is `bytes`.
+                If scannable is a file path, defaults to the file's basename.
+            rules_label (Optional[str], optional): Label for the ruleset. Defaults to the patterns joined by comma.
+            pattern_label (Optional[str], optional): Label for each pattern in the YARA rules. Defaults to "pattern".
+            regex_modifier (Optional[str], optional): Optional regex modifier (e.g. "nocase", "ascii", "wide", etc).
+                Only valid if `patterns_type` is `"regex"`.
+        """
+        rule_strings = []
+
+        for i, pattern in enumerate(patterns):
+            suffix = f"_{i + 1}" if len(patterns) > 1 else ''
+
+            rule_strings.append(yara_rule_string(
+                pattern=pattern,
+                pattern_type=patterns_type,
+                rule_name=f"{rules_label or YARALYZE}{suffix}",
+                pattern_label=f"{pattern_label}{suffix}" if pattern_label else None,
+                modifier=regex_modifier
+            ))
+
+        rules_string = newline_join(rule_strings)
+        rules_label = comma_join(patterns)
+        return cls(rules_string, rules_label, scannable, scannable_label)
+
+    def yaralyze(self) -> None:
+        """Use YARA to find matches and then force decode them."""
+        console.print(self)
+
+    def match_iterator(self) -> Iterator[Tuple[BytesMatch, BytesDecoder]]:
+        """
+        Iterator version of `yaralyze()`.
+
+        Yields:
+            Tuple[BytesMatch, BytesDecoder]: Match and decode data tuple.
+        """
+        self.rules.match(data=self.bytes, callback=self._yara_callback)
+
+        for yara_match in self.matches:
+            console.print(yara_match)
+            console.line()
+
+            for match in BytesMatch.from_yara_match(self.bytes, yara_match.match, self.highlight_style):
+                decoder = BytesDecoder(match, yara_match.rule_name)
+                self.extraction_stats.tally_match(decoder)
+                yield match, decoder
+
+        self._print_non_matches()
+
+    def _yara_callback(self, data: dict) -> Callable:
+        """
+        Callback invoked by `yara-python` to handle matches and non-matches as they are discovered.
+
+        Args:
+            data (dict): Data provided when `yara-python` invokes the callback.
+
+        Returns:
+            Callable: Always returns `yara.CALLBACK_CONTINUE` to signal `yara-python` should continue processing.
+        """
+        if data['matches']:
+            self.matches.append(YaraMatch(data, self._panel_text()))
+        else:
+            self.non_matches.append(data)
+
+        return yara.CALLBACK_CONTINUE
+
+    def _print_non_matches(self) -> None:
+        """Print info about the YARA rules that didn't match the bytes."""
+        if len(self.non_matches) == 0:
+            return
+
+        non_matches_text = sorted([Text(nm['rule'], 'grey') for nm in self.non_matches], key=str)
+
+        # Only show the non matches if there were valid ones, otherwise just show the number
+        if len(self.matches) == 0:
+            non_match_desc = f" did not match any of the {len(self.non_matches)} yara rules"
+            console.print(dim_if(self.__text__() + Text(non_match_desc, style='grey'), True))
+            return
+
+        non_match_desc = f" did not match the other {len(self.non_matches)} yara rules"
+        console.print(self.__text__() + Text(non_match_desc, style='grey') + Text(': '), style='dim')
+        console.print(Padding(Text(', ', 'white').join(non_matches_text), (0, 0, 1, 4)))
+
+    def _panel_text(self) -> Text:
+        """Inverted colors for the panel at the top of the match section of the output."""
+        styles = [reverse_color(YARALYZER_THEME.styles[f"yara.{s}"]) for s in ('scanned', 'rules')]
+        return self.__text__(*styles)
+
+    def _filename_string(self) -> str:
+        """The string to use when exporting this yaralyzer to SVG/HTML/etc."""
+        return str(self).replace('>', '').replace('<', '').replace(' ', '_')
+
+    def __text__(self, byte_style: Style | str = 'yara.scanned', rule_style: Style | str = 'yara.rules') -> Text:
+        """Text representation of this YARA scan (__text__() was taken)."""
+        txt = Text('').append(self.scannable_label, style=byte_style or 'yara.scanned')
+        return txt.append(' scanned with <').append(self.rules_label, style=rule_style or 'yara.rules').append('>')
+
+    def __rich_console__(self, _console: Console, options: ConsoleOptions) -> RenderResult:
+        """Does the stuff. TODO: not the best place to put the core logic."""
+        yield bytes_hashes_table(self.bytes, self.scannable_label)
+
+        for _bytes_match, bytes_decoder in self.match_iterator():
+            for attempt in bytes_decoder.__rich_console__(_console, options):
+                yield attempt
+
+    def __str__(self) -> str:
+        """Plain text (no rich colors) representation of the scan for display."""
+        return self.__text__().plain