yaralyzer 1.0.11__py3-none-any.whl

yaralyzer/output/file_export.py

@@ -0,0 +1,111 @@
+"""
+Functions to export Yaralyzer results to various file formats.
+"""
+import json
+import time
+from os import path
+from typing import Callable, Optional
+
+from rich.terminal_theme import TerminalTheme
+
+from yaralyzer.util.logging import log_and_print
+from yaralyzer.yaralyzer import Yaralyzer
+
+# TerminalThemes are used when saving SVGS. This one just swaps white for black in DEFAULT_TERMINAL_THEME
+YARALYZER_TERMINAL_THEME = TerminalTheme(
+    (0, 0, 0),
+    (255, 255, 255),
+    [
+        (0, 0, 0),
+        (128, 0, 0),
+        (0, 128, 0),
+        (128, 128, 0),
+        (0, 0, 128),
+        (128, 0, 128),
+        (0, 128, 128),
+        (192, 192, 192),
+    ],
+    [
+        (128, 128, 128),
+        (255, 0, 0),
+        (0, 255, 0),
+        (255, 255, 0),
+        (0, 0, 255),
+        (255, 0, 255),
+        (0, 255, 255),
+        (255, 255, 255),
+    ],
+)
+
+# Keys are export function names, values are options we always want to use w/that export function
+# Not meant for direct access; instead call invoke_rich_export().
+_EXPORT_KWARGS = {
+    'save_html': {
+        'inline_styles': True,
+        'theme': YARALYZER_TERMINAL_THEME,
+    },
+    'save_svg': {
+        'theme': YARALYZER_TERMINAL_THEME,
+    },
+    'save_text': {
+        'styles': True,
+    },
+}
+
+
+def export_json(yaralyzer: Yaralyzer, output_basepath: Optional[str]) -> str:
+    """
+    Export YARA scan results to JSON.
+
+    Args:
+        yaralyzer (Yaralyzer): The `Yaralyzer` object containing the results to export.
+        output_basepath (Optional[str]): Base path to write output to. Should have no file extension.
+
+    Returns:
+        str: Path data was exported to.
+    """
+    output_path = f"{output_basepath or 'yara_matches'}.json"
+
+    matches_data = [
+        bytes_match.to_json()
+        for bytes_match, _bytes_decoder in yaralyzer.match_iterator()
+    ]
+
+    with open(output_path, 'w') as f:
+        json.dump(matches_data, f, indent=4)
+
+    log_and_print(f"YARA matches exported to JSON file: '{output_path}'")
+    return output_path
+
+
+def invoke_rich_export(export_method: Callable, output_file_basepath: str) -> str:
+    """
+    Announce the export, perform the export, and announce completion.
+
+    Args:
+        export_method (Callable): Usually a `Rich.console.save_whatever()` method
+        output_file_basepath (str): Path to write output to. Should have no file extension.
+
+    Returns:
+        str: Path data was exported to.
+    """
+    method_name = export_method.__name__
+    extname = 'txt' if method_name == 'save_text' else method_name.split('_')[-1]
+    output_file_path = f"{output_file_basepath}.{extname}"
+
+    if method_name not in _EXPORT_KWARGS:
+        raise RuntimeError(f"{method_name} is not a valid Rich.console export method!")
+
+    kwargs = _EXPORT_KWARGS[method_name].copy()
+    kwargs.update({'clear': False})
+
+    if 'svg' in method_name:
+        kwargs.update({'title': path.basename(output_file_path)})
+
+    # Invoke it
+    log_and_print(f"Invoking Rich.console.{method_name}('{output_file_path}') with kwargs: '{kwargs}'...")
+    start_time = time.perf_counter()
+    export_method(output_file_path, **kwargs)
+    elapsed_time = time.perf_counter() - start_time
+    log_and_print(f"'{output_file_path}' written in {elapsed_time:02f} seconds")
+    return output_file_path

yaralyzer/output/file_hashes_table.py

@@ -0,0 +1,82 @@
+"""
+Methods for computing and displaying various file hashes.
+"""
+import hashlib
+from collections import namedtuple
+from typing import Optional, Union
+
+from rich.table import Column, Table
+
+from yaralyzer.helpers.rich_text_helper import LEFT, size_text
+from yaralyzer.output.rich_console import GREY
+
+BytesInfo = namedtuple('BytesInfo', ['size', 'md5', 'sha1', 'sha256'])
+
+
+def bytes_hashes_table(
+    bytes_or_bytes_info: Union[bytes, BytesInfo],
+    title: Optional[str] = None,
+    title_justify: str = LEFT
+) -> Table:
+    """
+    Build a Rich `Table` displaying the size, MD5, SHA1, and SHA256 hashes of a byte sequence.
+
+    Args:
+        bytes_or_bytes_info (Union[bytes, BytesInfo]): The `bytes` to hash, or a `BytesInfo`
+            namedtuple with precomputed values.
+        title (Optional[str], optional): Optional title for the table. Defaults to `None`.
+        title_justify (str, optional): Justification for the table title. Defaults to `"LEFT"`.
+
+    Returns:
+        Table: A Rich `Table` object with the size and hash values.
+    """
+    if isinstance(bytes_or_bytes_info, bytes):
+        bytes_info = compute_file_hashes(bytes_or_bytes_info)
+    else:
+        bytes_info = bytes_or_bytes_info
+
+    table = Table(
+        'Size',
+        Column(size_text(bytes_info.size)),
+        title=f" {title} Bytes Info" if title else None,
+        title_style=GREY,
+        title_justify=title_justify
+    )
+    table.add_row('MD5', bytes_info.md5)
+    table.add_row('SHA1', bytes_info.sha1)
+    table.add_row('SHA256', bytes_info.sha256)
+    table.columns[1].style = 'orange3'
+    table.columns[1].header_style = 'bright_cyan'
+    return table
+
+
+def compute_file_hashes(_bytes: bytes) -> BytesInfo:
+    """
+    Compute the size, MD5, SHA1, and SHA256 hashes for a given byte sequence.
+
+    Args:
+        _bytes (bytes): The `bytes` to hash.
+
+    Returns:
+        BytesInfo: `BytesInfo` namedtuple containing size, md5, sha1, and sha256 values.
+    """
+    return BytesInfo(
+        size=len(_bytes),
+        md5=hashlib.md5(_bytes).hexdigest().upper(),
+        sha1=hashlib.sha1(_bytes).hexdigest().upper(),
+        sha256=hashlib.sha256(_bytes).hexdigest().upper()
+    )
+
+
+def compute_file_hashes_for_file(file_path) -> BytesInfo:
+    """
+    Compute the size, MD5, SHA1, and SHA256 hashes for the contents of a file.
+
+    Args:
+        file_path (str): Path to the file to hash.
+
+    Returns:
+        BytesInfo: `BytesInfo` namedtuple containing size, md5, sha1, and sha256 values for the file contents.
+    """
+    with open(file_path, 'rb') as file:
+        return compute_file_hashes(file.read())

yaralyzer/output/regex_match_metrics.py

@@ -0,0 +1,97 @@
+"""
+`RegexMatchMetrics` class.
+"""
+from collections import defaultdict
+
+from yaralyzer.decoding.bytes_decoder import BytesDecoder
+from yaralyzer.util.logging import log
+
+
+class RegexMatchMetrics:
+    """
+    Class to measure what we enounter as we iterate over all matches of a relatively simple byte level regex.
+
+    Things like how much many of our matched bytes were we able to decode easily vs. by force vs. not at all,
+    were some encodings have a higher pct of success than others (indicating part of our mystery data might be
+    encoded that way?
+
+    Example:
+        "Find bytes between quotes" against a relatively large pool of close to random encrypted binary data.
+
+    Attributes:
+        match_count (int): Total number of matches found.
+        bytes_matched (int): Total number of bytes matched across all matches.
+        matches_decoded (int): Number of matches where we were able to decode at least some of the matched bytes.
+        easy_decode_count (int): Number of matches where we were able to decode the matched bytes without forcing.
+        forced_decode_count (int): Number of matches where we were only able to decode the matched bytes by forcing.
+        undecodable_count (int): Number of matches where we were unable to decode any of the matched bytes.
+        skipped_matches_lengths (defaultdict): Dictionary mapping lengths of skipped matches to their counts.
+        bytes_match_objs (list): List of `BytesMatch` objects for all matches encountered.
+        per_encoding_stats (defaultdict): Dictionary mapping encoding names to their respective `RegexMatchMetrics`.
+
+    TODO: use @dataclass decorator https://realpython.com/python-data-classes/
+    """
+
+    def __init__(self) -> None:
+        self.match_count = 0
+        self.bytes_matched = 0
+        self.matches_decoded = 0
+        self.easy_decode_count = 0
+        self.forced_decode_count = 0
+        self.undecodable_count = 0
+        self.skipped_matches_lengths = defaultdict(lambda: 0)
+        self.bytes_match_objs = []  # Keep a copy of all matches in memory
+        self.per_encoding_stats = defaultdict(lambda: RegexMatchMetrics())
+
+    def num_matches_skipped_for_being_empty(self) -> int:
+        """Number of matches skipped for being empty (0 length)."""
+        return self.skipped_matches_lengths[0]
+
+    def num_matches_skipped_for_being_too_big(self) -> int:
+        """Number of matches skipped for being too big to decode."""
+        return sum({k: v for k, v in self.skipped_matches_lengths.items() if k > 0}.values())
+
+    def tally_match(self, decoder: BytesDecoder) -> None:
+        """
+        Tally statistics from a `BytesDecoder` after it has processed a match.
+
+        Args:
+            decoder (BytesDecoder): The `BytesDecoder` that processed a match.
+        """
+        log.debug(f"Tallying {decoder.bytes_match} ({len(decoder.decodings)} decodings)")
+        self.match_count += 1
+        self.bytes_matched += decoder.bytes_match.match_length
+        self.bytes_match_objs.append(decoder.bytes_match)
+
+        if not decoder.bytes_match.is_decodable():
+            self.skipped_matches_lengths[decoder.bytes_match.match_length] += 1
+
+        for decoding_attempt in decoder.decodings:
+            log.debug(f"Tallying decoding for {decoding_attempt.encoding}")
+            encoding_stats = self.per_encoding_stats[decoding_attempt.encoding]
+
+            if decoding_attempt.failed_to_decode:
+                encoding_stats.undecodable_count += 1
+            else:
+                encoding_stats.match_count += 1
+                encoding_stats.matches_decoded += 1
+
+                if decoding_attempt.was_force_decoded:
+                    encoding_stats.forced_decode_count += 1
+                else:
+                    encoding_stats.easy_decode_count += 1
+
+    def __eq__(self, other):
+        for k, v in vars(self).items():
+            if v != vars(other)[k]:
+                return False
+
+        return True
+
+    def __str__(self):
+        return f"<matches: {self.match_count}, " + \
+               f"bytes: {self.bytes_matched}, " + \
+               f"decoded: {self.matches_decoded} " + \
+               f"too_big: {self.num_matches_skipped_for_being_too_big()}, " + \
+               f"empty: {self.num_matches_skipped_for_being_empty()}>" + \
+               f"empty: {self.undecodable_count}>"

yaralyzer/output/rich_console.py

@@ -0,0 +1,114 @@
+"""
+Variables and methods for working with Rich text output.
+"""
+from shutil import get_terminal_size
+from typing import List
+
+from rich.console import Console
+from rich.errors import MarkupError
+from rich.style import Style
+from rich.text import Text
+from rich.theme import Theme
+
+from yaralyzer.config import is_env_var_set_and_not_false, is_invoked_by_pytest
+
+# Colors
+ALERT_STYLE = 'error'  # Regex Capture used when extracting quoted chunks of bytes
+BYTES = 'color(100) dim'
+BYTES_NO_DIM = 'color(100)'
+BYTES_BRIGHTEST = 'color(220)'
+BYTES_BRIGHTER = 'orange1'
+BYTES_HIGHLIGHT = 'color(136)'
+DANGER_HEADER = 'color(88) on white'  # Red
+DARK_GREY = 'color(236)'
+GREY = 'color(241)'
+GREY_ADDRESS = 'color(238)'
+PEACH = 'color(215)'
+
+# Theme used by main console
+YARALYZER_THEME_DICT = {
+    # colors
+    'dark_orange': 'color(58)',
+    'grey': GREY,
+    'grey.dark': DARK_GREY,
+    'grey.dark_italic': f"{DARK_GREY} italic",
+    'grey.darker_italic': 'color(8) dim italic',
+    'grey.darkest': 'color(235) dim',
+    'grey.light': 'color(248)',
+    'off_white': 'color(245)',
+    'zero_bytes': 'color(20)',
+    # data types
+    'encoding': 'color(158) underline bold',
+    'encoding.header': 'color(158) bold',
+    'encoding.language': 'dark_green italic',
+    'number': 'cyan',
+    'regex': 'color(218) dim',
+    'no_attempt': "color(60) dim italic",
+    # design elements
+    'decode.section_header': 'color(100) reverse',
+    'decode.subheading': PEACH,
+    'decode.subheading_2': 'color(215) dim italic',
+    'decode.table_header': 'color(101) bold',
+    'headline': 'bold white underline',
+    # bytes
+    'ascii': 'color(58)',
+    'ascii_unprintable': 'color(131)',
+    'bytes': BYTES,
+    'bytes.title_dim': 'orange1 dim',
+    'bytes.title': BYTES_BRIGHTER,
+    'bytes.decoded': BYTES_BRIGHTEST,
+    # yara
+    'matched_rule': 'on bright_black bold',
+    'yara.key': DARK_GREY,
+    'yara.match_var': 'color(156) italic',
+    'yara.string': 'white',
+    'yara.date': 'color(216)',
+    'yara.url': 'color(220)',
+    'yara.int': 'color(45)',
+    'yara.hex': 'color(98)',
+    'yara.scanned': Style(color='yellow', underline=True, bold=True),
+    'yara.rules':  Style(color='color(135)', underline=True, bold=True),
+    # error log events
+    'error': 'bright_red',
+}
+
+YARALYZER_THEME = Theme(YARALYZER_THEME_DICT)
+DEFAULT_CONSOLE_WIDTH = 160
+
+
+def console_width_possibilities():
+    """Returns a list of possible console widths, the first being the current terminal width."""
+    # Subtract 2 from terminal cols just as a precaution in case things get weird
+    return [get_terminal_size().columns - 2, DEFAULT_CONSOLE_WIDTH]
+
+
+# Maximize output width if YARALYZER_MAXIMIZE_WIDTH is set (also can changed with --maximize-width option)
+if is_invoked_by_pytest():
+    CONSOLE_WIDTH = DEFAULT_CONSOLE_WIDTH
+elif is_env_var_set_and_not_false('YARALYZER_MAXIMIZE_WIDTH'):
+    CONSOLE_WIDTH = max(console_width_possibilities())
+else:
+    CONSOLE_WIDTH = min(console_width_possibilities())
+
+# Many bytes take 4 chars to print (e.g. '\xcc') so this is the max bytes we can safely print in a line
+CONSOLE_PRINT_BYTE_WIDTH = int(CONSOLE_WIDTH / 4.0)
+console = Console(theme=YARALYZER_THEME, color_system='256', highlight=False, width=CONSOLE_WIDTH)
+
+
+def console_print_with_fallback(_string: Text | str, style=None) -> None:
+    """`rich.console.print()` with fallback to regular `print()` if there's a Rich Markup issue."""
+    try:
+        console.print(_string, style=style)
+    except MarkupError:
+        console.print(f"Hit a bracket issue with rich.console printing, defaulting to plain print", style='warn')
+        print(_string.plain if isinstance(_string, Text) else _string)
+
+
+def console_width() -> int:
+    """Current width set in `console` object."""
+    return console._width or 40
+
+
+def theme_colors_with_prefix(prefix: str) -> List[Text]:
+    """Return a list of (name, style) `Text` objects for all styles in the theme that start with `prefix`."""
+    return [Text(k, v) for k, v in YARALYZER_THEME.styles.items() if k.startswith(prefix)]