yaralyzer 1.0.7__py3-none-any.whl → 1.0.8__py3-none-any.whl

CHANGELOG.md

@@ -1,8 +1,14 @@
 # NEXT RELEASE
 
+### 1.0.8
+* Bump `python-dotenv` to v1.1.1
+* Use `mkdocs` and `lazydocs` to build automatic API documentation at https://michelcrypt4d4mus.github.io/yaralyzer/
+* Drop python 3.9 support (required by `mkdocs-awesome-nav` package)
+
 ### 1.0.7
 * Add `Changelog` to PyPi URLs, add some more PyPi classifiers
 * Add `.flake8` config file and fix style errors
+* Rename `prefix_with_plain_text_obj()` to `prefix_with_style()`
 
 ### 1.0.6
 * Add `Environment :: Console` and `Programming Language :: Python` to PyPi classifiers

yaralyzer/__init__.py

@@ -24,6 +24,11 @@ PDFALYZER_MSG_TXT.append('https://github.com/michelcrypt4d4mus/pdfalyzer\n', sty
 
 
 def yaralyze():
+    """
+    Entry point for yaralyzer when invoked as a script.
+
+    Args are parsed from the command line and environment variables. See yaralyzer --help for details.
+    """
     args = parse_arguments()
     output_basepath = None
 

yaralyzer/bytes_match.py

@@ -1,10 +1,4 @@
-"""
-Simple class to keep track of regex matches against binary data.  Basically an re.match object with
-some (not many) extra bells and whistles, most notably the surrounding_bytes property.
-
-pre_capture_len and post_capture_len refer to the regex sections before and after the capture group,
-e.g. a regex like '123(.*)x:' would have pre_capture_len of 3 and post_capture_len of 2.
-"""
+"""BytesMatch class for tracking regex and YARA matches against binary data."""
 import re
 from typing import Iterator, Optional
 
@@ -19,6 +13,16 @@ from yaralyzer.output.rich_console import ALERT_STYLE, GREY_ADDRESS
 
 
 class BytesMatch:
+    """
+    Simple class to keep track of regex matches against binary data.
+
+    Basically an re.match object with some (not many) extra bells and whistles, most notably
+    the surrounding_bytes property.
+
+    pre_capture_len and post_capture_len refer to the regex sections before and after the capture group,
+    e.g. a regex like '123(.*)x:' would have pre_capture_len of 3 and post_capture_len of 2.
+    """
+
     def __init__(
         self,
         matched_against: bytes,
@@ -30,8 +34,16 @@ class BytesMatch:
         highlight_style: str = YaralyzerConfig.HIGHLIGHT_STYLE
     ) -> None:
         """
-        Ordinal means it's the Nth match with this regex (not super important but useful)
-        YARA makes it a little rouch to get the actual regex that matched. Can be done with plyara eventually.
+        Initialize a BytesMatch object representing a match against binary data.
+
+        Args:
+            matched_against (bytes): The full byte sequence that was searched.
+            start_idx (int): Start index of the match in the byte sequence.
+            length (int): Length of the match in bytes.
+            label (str): Label for the match (e.g., regex or YARA rule name).
+            ordinal (int): The Nth match for this pattern.
+            match (Optional[re.Match]): Regex match object, if available.
+            highlight_style (str): Style to use for highlighting the match.
         """
         self.matched_against: bytes = matched_against
         self.start_idx: int = start_idx
@@ -58,6 +70,18 @@ class BytesMatch:
         ordinal: int,
         highlight_style: str = YaralyzerConfig.HIGHLIGHT_STYLE
     ) -> 'BytesMatch':
+        """
+        Create a BytesMatch from a regex match object.
+
+        Args:
+            matched_against (bytes): The bytes searched.
+            match (re.Match): The regex match object.
+            ordinal (int): The Nth match for this pattern.
+            highlight_style (str): Style for highlighting.
+
+        Returns:
+            BytesMatch: The constructed BytesMatch instance.
+        """
         return cls(matched_against, match.start(), len(match[0]), match.re.pattern, ordinal, match, highlight_style)
 
     @classmethod
@@ -70,7 +94,20 @@ class BytesMatch:
         ordinal: int,
         highlight_style: str = YaralyzerConfig.HIGHLIGHT_STYLE
     ) -> 'BytesMatch':
-        """Build a BytesMatch from a yara string match. 'matched_against' is the set of bytes yara was run against."""
+        """
+        Build a BytesMatch from a YARA string match instance.
+
+        Args:
+            matched_against (bytes): The bytes searched.
+            rule_name (str): Name of the YARA rule.
+            yara_str_match (StringMatch): YARA string match object.
+            yara_str_match_instance (StringMatchInstance): Instance of the string match.
+            ordinal (int): The Nth match for this pattern.
+            highlight_style (str): Style for highlighting.
+
+        Returns:
+            BytesMatch: The constructed BytesMatch instance.
+        """
         pattern_label = yara_str_match.identifier
 
         # Don't duplicate the labeling if rule_name and yara_str are the same
@@ -94,7 +131,17 @@ class BytesMatch:
         yara_match: dict,
         highlight_style: str = YaralyzerConfig.HIGHLIGHT_STYLE
     ) -> Iterator['BytesMatch']:
-        """Iterator w/a BytesMatch for each string returned as part of a YARA match result dict."""
+        """
+        Yield a BytesMatch for each string returned as part of a YARA match result dict.
+
+        Args:
+            matched_against (bytes): The bytes searched.
+            yara_match (dict): YARA match result dictionary.
+            highlight_style (str): Style for highlighting.
+
+        Yields:
+            BytesMatch: For each string match in the YARA result.
+        """
         i = 0  # For numbered labeling
 
         # yara-python's internals changed with 4.3.0: https://github.com/VirusTotal/yara-python/releases/tag/v4.3.0
@@ -112,14 +159,27 @@ class BytesMatch:
                 )
 
     def style_at_position(self, idx) -> str:
-        """Get the style for the byte at position idx within the matched bytes"""
+        """
+        Get the style for the byte at position idx within the matched bytes.
+
+        Args:
+            idx (int): Index within the surrounding bytes.
+
+        Returns:
+            str: The style to use for this byte (highlight or greyed out).
+        """
         if idx < self.highlight_start_idx or idx >= self.highlight_end_idx:
             return GREY_ADDRESS
         else:
             return self.highlight_style
 
     def location(self) -> Text:
-        """Returns a Text obj like '(start idx: 348190, end idx: 348228)'"""
+        """
+        Get a styled Text object describing the start and end index of the match.
+
+        Returns:
+            Text: Rich Text object like '(start idx: 348190, end idx: 348228)'.
+        """
         location_txt = prefix_with_style(
             f"(start idx: ",
             style='off_white',
@@ -133,13 +193,26 @@ class BytesMatch:
         return location_txt
 
     def is_decodable(self) -> bool:
-        """True if SUPPRESS_DECODES_TABLE is false and length of self.bytes is between MIN/MAX_DECODE_LENGTH"""
+        """
+        Determine if the matched bytes should be decoded.
+
+        Whether the bytes are decodable depends on whether SUPPRESS_DECODES_TABLE is set
+        and whether the match length is between MIN/MAX_DECODE_LENGTH.
+
+        Returns:
+            bool: True if decodable, False otherwise.
+        """
         return self.match_length >= YaralyzerConfig.args.min_decode_length \
            and self.match_length <= YaralyzerConfig.args.max_decode_length \
            and not YaralyzerConfig.args.suppress_decodes_table
 
     def bytes_hashes_table(self) -> Table:
-        """Helper function to build the MD5/SHA table for self.bytes"""
+        """
+        Build a table of MD5/SHA hashes for the matched bytes.
+
+        Returns:
+            Table: Rich Table object with hashes.
+        """
         return bytes_hashes_table(
             self.bytes,
             self.location().plain,
@@ -147,7 +220,12 @@ class BytesMatch:
         )
 
     def suppression_notice(self) -> Text:
-        """Generate a message for when there are too few/too many bytes"""
+        """
+        Generate a message for when the match is too short or too long to decode.
+
+        Returns:
+            Text: Rich Text object with the suppression notice.
+        """
         txt = self.__rich__()
 
         if self.match_length < YaralyzerConfig.args.min_decode_length:
@@ -159,7 +237,12 @@ class BytesMatch:
         return txt
 
     def to_json(self) -> dict:
-        """Convert this BytesMatch to a JSON-serializable dict."""
+        """
+        Convert this BytesMatch to a JSON-serializable dictionary.
+
+        Returns:
+            dict: Dictionary representation of the match, suitable for JSON serialization.
+        """
         json_dict = {
             'label': self.label,
             'match_length': self.match_length,
@@ -178,7 +261,13 @@ class BytesMatch:
         return json_dict
 
     def _find_surrounding_bytes(self, num_before: Optional[int] = None, num_after: Optional[int] = None) -> None:
-        """Find the surrounding bytes, making sure not to step off the beginning or end"""
+        """
+        Find and set the bytes surrounding the match, ensuring indices stay within bounds.
+
+        Args:
+            num_before (Optional[int]): Number of bytes before the match to include.
+            num_after (Optional[int]): Number of bytes after the match to include.
+        """
         num_after = num_after or num_before or YaralyzerConfig.args.surrounding_bytes
         num_before = num_before or YaralyzerConfig.args.surrounding_bytes
         self.surrounding_start_idx: int = max(self.start_idx - num_before, 0)
@@ -186,6 +275,7 @@ class BytesMatch:
         self.surrounding_bytes: bytes = self.matched_against[self.surrounding_start_idx:self.surrounding_end_idx]
 
     def __rich__(self) -> Text:
+        """Get a rich Text representation of the match for display."""
         headline = prefix_with_style(str(self.match_length), style='number', root_style='decode.subheading')
         headline.append(f" bytes matching ")
         headline.append(f"{self.label} ", style=ALERT_STYLE if self.highlight_style == ALERT_STYLE else 'regex')
@@ -193,4 +283,5 @@ class BytesMatch:
         return headline + self.location()
 
     def __str__(self):
+        """Plain text (no rich colors) representation of the match for display."""
         return self.__rich__().plain

yaralyzer/config.py

@@ -1,3 +1,6 @@
+"""
+Configuration management for Yaralyzer.
+"""
 import logging
 from argparse import ArgumentParser, Namespace
 from os import environ
@@ -15,16 +18,19 @@ MEGABYTE = 1024 * KILOBYTE
 
 def config_var_name(env_var: str) -> str:
     """
-    Get the name of env_var and strip off 'YARALYZER_', e.g.:
-        SURROUNDING_BYTES_ENV_VAR = 'YARALYZER_SURROUNDING_BYTES'
-        config_var_name(SURROUNDING_BYTES_ENV_VAR) => 'SURROUNDING_BYTES'
+    Get the name of env_var and strip off 'YARALYZER_' prefix.
+
+    Example:
+        $ SURROUNDING_BYTES_ENV_VAR = 'YARALYZER_SURROUNDING_BYTES'
+        $ config_var_name(SURROUNDING_BYTES_ENV_VAR) => 'SURROUNDING_BYTES'
+
     """
     env_var = env_var.removeprefix("YARALYZER_")
     return f'{env_var=}'.partition('=')[0]
 
 
 def is_env_var_set_and_not_false(var_name):
-    """Returns True if var_name is not empty and set to anything other than 'false' (capitalization agnostic)"""
+    """Return True if var_name is not empty and set to anything other than 'false' (capitalization agnostic)."""
     if var_name in environ:
         var_value = environ[var_name]
         return var_value is not None and len(var_value) > 0 and var_value.lower() != 'false'
@@ -33,11 +39,13 @@ def is_env_var_set_and_not_false(var_name):
 
 
 def is_invoked_by_pytest():
-    """Return true if pytest is running"""
+    """Return true if pytest is running."""
     return is_env_var_set_and_not_false(PYTEST_FLAG)
 
 
 class YaralyzerConfig:
+    """Handles parsing of command line args and environment variables for Yaralyzer."""
+
     # Passed through to yara.set_config()
     DEFAULT_MAX_MATCH_LENGTH = 100 * KILOBYTE
     DEFAULT_YARA_STACK_SIZE = 2 * 65536
@@ -76,11 +84,13 @@ class YaralyzerConfig:
 
     @classmethod
     def set_argument_parser(cls, parser: ArgumentParser) -> None:
+        """Sets the _argument_parser instance variable that will be used to parse command line args."""
         cls._argument_parser: ArgumentParser = parser
         cls._argparse_keys: List[str] = sorted([action.dest for action in parser._actions])
 
     @classmethod
     def set_args(cls, args: Namespace) -> None:
+        """Set the args class instance variable and update args with any environment variable overrides."""
         cls.args = args
 
         for option in cls._argparse_keys:
@@ -105,9 +115,11 @@ class YaralyzerConfig:
 
     @classmethod
     def set_default_args(cls):
+        """Set args to their defaults as if parsed from the command line."""
         cls.set_args(cls._argument_parser.parse_args(['dummy']))
 
     @classmethod
     def get_default_arg(cls, arg: str) -> Any:
+        """Return the default value for arg as defined by a DEFAULT_ style class variable."""
         default_var = f"DEFAULT_{arg.upper()}"
         return vars(cls).get(default_var)

yaralyzer/decoding/bytes_decoder.py

@@ -1,8 +1,4 @@
-"""
-Class to handle attempting to decode a chunk of bytes into strings with various possible encodings.
-Leverages the chardet library to both guide what encodings are attempted as well as to rank decodings
-in the results.
-"""
+"""BytesDecoder class for attempting to decode bytes with various encodings."""
 from collections import defaultdict
 from copy import deepcopy
 from operator import attrgetter
@@ -34,7 +30,33 @@ SCORE_SCALER = 100.0
 
 
 class BytesDecoder:
+    """
+    Class to handle attempting to decode a chunk of bytes into strings with various possible encodings.
+
+    Leverages the chardet library to both guide what encodings are attempted as well as to rank decodings
+    in the results.
+    """
+
     def __init__(self, bytes_match: 'BytesMatch', label: Optional[str] = None) -> None:
+        """
+        Initialize a BytesDecoder for attempting to decode a chunk of bytes using various encodings.
+
+        Args:
+            bytes_match (BytesMatch): The BytesMatch object containing the bytes to decode and match metadata.
+            label (Optional[str], optional): Optional label for this decoding attempt. Defaults to the match label.
+
+        Attributes:
+            bytes_match (BytesMatch): The BytesMatch instance being decoded.
+            bytes (bytes): The bytes (including surrounding context) to decode.
+            label (str): Label for this decoding attempt.
+            was_match_decodable (dict): Tracks successful decodes per encoding.
+            was_match_force_decoded (dict): Tracks forced decodes per encoding.
+            was_match_undecodable (dict): Tracks failed decodes per encoding.
+            decoded_strings (dict): Maps encoding to decoded string.
+            undecoded_rows (list): Stores undecoded table rows.
+            decodings (list): List of DecodingAttempt objects for each encoding tried.
+            encoding_detector (EncodingDetector): Used to detect and assess possible encodings.
+        """
         self.bytes_match = bytes_match
         self.bytes = bytes_match.surrounding_bytes
         self.label = label or bytes_match.label
@@ -51,7 +73,7 @@ class BytesDecoder:
         self.encoding_detector = EncodingDetector(self.bytes)
 
     def __rich_console__(self, _console: Console, options: ConsoleOptions) -> RenderResult:
-        """Rich object generator (see Rich console docs)"""
+        """Rich object generator (see Rich console docs)."""
         yield NewLine(2)
         yield Align(self._decode_attempt_subheading(), CENTER)
 
@@ -70,7 +92,7 @@ class BytesDecoder:
         yield Align(self.bytes_match.bytes_hashes_table(), CENTER, style='dim')
 
     def _build_decodings_table(self, suppress_decodes: bool = False) -> Table:
-        """First rows are the raw / hex views of the bytes, next rows are the attempted decodings"""
+        """First rows are the raw / hex views of the bytes, next rows are the attempted decodings."""
         self.table = new_decoding_attempts_table(self.bytes_match)
 
         # Add the encoding rows to the table if not suppressed
@@ -115,7 +137,7 @@ class BytesDecoder:
         return Panel(headline, style='decode.subheading', expand=False)
 
     def _track_decode_stats(self) -> None:
-        """Track stats about successful vs. forced vs. failed decode attempts"""
+        """Track stats about successful vs. forced vs. failed decode attempts."""
         for decoding in self.decodings:
             if decoding.failed_to_decode:
                 self.was_match_undecodable[decoding.encoding] += 1
@@ -162,7 +184,7 @@ class BytesDecoder:
 
 
 def _build_encodings_metric_dict():
-    """One key for each key in ENCODINGS_TO_ATTEMPT, values are all 0"""
+    """One key for each key in ENCODINGS_TO_ATTEMPT, values are all 0."""
     metrics_dict = defaultdict(lambda: 0)
 
     for encoding in ENCODINGS_TO_ATTEMPT.keys():

yaralyzer/decoding/decoding_attempt.py

@@ -1,13 +1,11 @@
-"""
-Class to manage attempting to decode a chunk of bytes into strings with a given encoding.
-"""
+"""Class to manage attempting to decode a chunk of bytes into strings with a given encoding."""
 from sys import byteorder
 from typing import Optional
 
 from rich.markup import escape
 from rich.text import Text
 
-from yaralyzer.bytes_match import BytesMatch  # Used to cause circular import issues
+from yaralyzer.bytes_match import BytesMatch  # Formerly caused circular import issues
 from yaralyzer.encoding_detection.character_encodings import (ENCODINGS_TO_ATTEMPT, SINGLE_BYTE_ENCODINGS,
      UTF_8, encoding_width, is_wide_utf)
 from yaralyzer.helpers.bytes_helper import clean_byte_string, truncate_for_encoding
@@ -17,6 +15,8 @@ from yaralyzer.util.logging import log
 
 
 class DecodingAttempt:
+    """Class to manage attempting to decode a chunk of bytes into strings with a given encoding."""
+
     def __init__(self, bytes_match: 'BytesMatch', encoding: str) -> None:
         # Args
         self.bytes = bytes_match.surrounding_bytes
@@ -31,7 +31,7 @@ class DecodingAttempt:
         self.decoded_string = self._decode_bytes()
 
     def is_wide_utf_encoding(self) -> bool:
-        """Returns True if the encoding is UTF-16 or UTF-32"""
+        """Returns True if the encoding is UTF-16 or UTF-32."""
         return is_wide_utf(self.encoding)
 
     def _decode_bytes(self) -> Text:
@@ -58,7 +58,7 @@ class DecodingAttempt:
             return self._custom_decode()
 
     def _custom_decode(self) -> Text:
-        """Returns a Text obj representing an attempt to force a UTF-8 encoding upon an array of bytes"""
+        """Returns a Text obj representing an attempt to force a UTF-8 encoding upon an array of bytes."""
         log.info(f"Custom decoding {self.bytes_match} with {self.encoding}...")
         unprintable_char_map = ENCODINGS_TO_ATTEMPT.get(self.encoding)
         output = Text('', style='bytes.decoded')
@@ -146,7 +146,7 @@ class DecodingAttempt:
             return self._failed_to_decode_msg_txt(last_exception)
 
     def _to_rich_text(self, _string: str, bytes_offset: int = 0) -> Text:
-        """Convert a decoded string to highlighted Text representation"""
+        """Convert a decoded string to highlighted Text representation."""
         # Adjust where we start the highlighting given the multibyte nature of the encodings
         log.debug(f"Stepping through {self.encoding} encoded string...")
         txt = Text('', style=self.bytes_match.style_at_position(0))

yaralyzer/encoding_detection/character_encodings.py

@@ -1,5 +1,6 @@
 """
-Constants related to character encodings
+Constants related to character encodings.
+
 * https://www.mit.edu/people/kenta/two/iso8859.html
 * https://www.utf8-chartable.de/unicode-utf8-table.pl?utf8=dec
 """

yaralyzer/encoding_detection/encoding_assessment.py

@@ -1,5 +1,5 @@
 """
-Class to smooth some of the rough edges around the dicts returned by chardet.detect_all()
+Help with chardet library.
 """
 from typing import Any, Optional
 
@@ -14,7 +14,13 @@ LANGUAGE = 'language'
 
 
 class EncodingAssessment:
+    """Class to smooth some of the rough edges around the dicts returned by chardet.detect_all()"""
+
     def __init__(self, assessment: dict) -> None:
+        """
+        Args:
+            assessment (dict): The dict returned by chardet.detect_all().
+        """
         self.assessment = assessment
         self.encoding = assessment[ENCODING].lower()
 
@@ -27,7 +33,7 @@ class EncodingAssessment:
         self.set_encoding_label(self.language.title() if self.language else None)
 
     @classmethod
-    def dummy_encoding_assessment(cls, encoding) -> 'EncodingAssessment':
+    def dummy_encoding_assessment(cls, encoding: str) -> 'EncodingAssessment':
         """Generate an empty EncodingAssessment to use as a dummy when chardet gives us nothing."""
         assessment = cls({ENCODING: encoding, CONFIDENCE: 0.0})
         assessment.confidence_text = Text('none', 'no_attempt')

yaralyzer/encoding_detection/encoding_detector.py

@@ -1,7 +1,4 @@
-"""
-Manager class to ease dealing with the chardet encoding detection library 'chardet'.
-Each instance of this class manages a chardet.detect_all() scan on a single set of bytes.
-"""
+"""EncodingDetector class for managing chardet encoding detection."""
 from operator import attrgetter
 from typing import List
 
@@ -18,10 +15,15 @@ CONFIDENCE_SCORE_RANGE = range(0, 101)
 
 
 class EncodingDetector:
-    # 10 as in 10%, 0.02, etc.  Encodings w/confidences below this will not be displayed in the decoded table
+    """
+    Manager class to ease dealing with the chardet encoding detection library 'chardet'.
+    Each instance of this class manages a chardet.detect_all() scan on a single set of bytes.
+    """
+
+    # Default value for encodings w/confidences below this will not be displayed in the decoded table
     force_display_threshold = 20.0
 
-    # At what chardet.detect() confidence % should we force a decode with an obscure encoding?
+    # Default value for what chardet.detect() confidence % should we force a decode with an obscure encoding.
     force_decode_threshold = 50.0
 
     def __init__(self, _bytes: bytes) -> None:
@@ -53,21 +55,23 @@ class EncodingDetector:
         self.force_display_assessments = self.assessments_above_confidence(type(self).force_display_threshold)
 
     def get_encoding_assessment(self, encoding: str) -> EncodingAssessment:
-        """If chardet produced one, return it, otherwise return a dummy node with confidence of 0"""
+        """If chardet produced one, return it, otherwise return a dummy node with confidence of 0."""
         assessment = next((r for r in self.unique_assessments if r.encoding == encoding), None)
         return assessment or EncodingAssessment.dummy_encoding_assessment(encoding)
 
     def has_enough_bytes(self) -> bool:
+        """Return true if we have enough bytes to run chardet.detect()."""
         return self.bytes_len >= YaralyzerConfig.args.min_chardet_bytes
 
     def assessments_above_confidence(self, cutoff: float) -> List[EncodingAssessment]:
+        """Return the assessments above the given confidence cutoff."""
         return [a for a in self.unique_assessments if a.confidence >= cutoff]
 
     def __rich__(self) -> Padding:
         return Padding(self.table, (0, 0, 0, 0))
 
     def _uniquify_results_and_build_table(self) -> None:
-        """Keep the highest result per encoding, ignoring the language chardet has indicated"""
+        """Keep the highest result per encoding, ignoring the language chardet has indicated."""
         already_seen_encodings = {}
 
         for i, result in enumerate(self.assessments):
@@ -87,6 +91,7 @@ class EncodingDetector:
         self.unique_assessments.sort(key=attrgetter('confidence'), reverse=True)
 
     def _set_empty_results(self) -> None:
+        """Set empty results for when chardet can't help us."""
         self.assessments = []
         self.unique_assessments = []
         self.raw_chardet_assessments = []
@@ -95,7 +100,7 @@ class EncodingDetector:
 
 
 def _empty_chardet_results_table():
-    """Returns a fresh table"""
+    """Returns a fresh table."""
     table = Table(
         'Rank', 'Encoding', 'Confidence',
         title='chardet.detect results',