simple-detect-secrets 0.0.1__py3-none-any.whl

simple_detect_secrets/__init__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+VERSION = version('simple-detect-secrets')

simple_detect_secrets/core/baseline.py

@@ -0,0 +1,249 @@
+import os
+import re
+import subprocess
+
+from simple_detect_secrets import util
+from simple_detect_secrets.core.log import get_logger
+from simple_detect_secrets.core.secrets_collection import SecretsCollection
+
+log = get_logger(format_string='%(message)s')
+
+
+def initialize(
+    path,
+    plugins,
+    exclude_files_regex=None,
+    exclude_lines_regex=None,
+    word_list_file=None,
+    word_list_hash=None,
+    should_scan_all_files=False,
+):
+    """Scans the entire codebase for secrets, and returns a
+    SecretsCollection object.
+
+    :type path: list
+
+    :type plugins: tuple of detect_secrets.plugins.base.BasePlugin
+    :param plugins: rules to initialize the SecretsCollection with.
+
+    :type exclude_files_regex: str|None
+    :type exclude_lines_regex: str|None
+
+    :type word_list_file: str|None
+    :param word_list_file: optional word list file for ignoring certain words.
+
+    :type word_list_hash: str|None
+    :param word_list_hash: optional iterated sha1 hash of the words in the word list.
+
+    :type should_scan_all_files: bool
+
+    :rtype: SecretsCollection
+    """
+    output = SecretsCollection(
+        plugins,
+        exclude_files=exclude_files_regex,
+        exclude_lines=exclude_lines_regex,
+        word_list_file=word_list_file,
+        word_list_hash=word_list_hash,
+    )
+
+    files_to_scan = []
+    for element in path:
+        if os.path.isdir(element):
+            if should_scan_all_files:
+                files_to_scan.extend(
+                    _get_files_recursively(element),
+                )
+            else:
+                files_to_scan.extend(
+                    _get_git_tracked_files(element),
+                )
+        elif os.path.isfile(element):
+            files_to_scan.append(element)
+        else:
+            log.error('detect-secrets: %s: No such file or directory', element)
+
+    if not files_to_scan:
+        return output
+
+    if exclude_files_regex:
+        exclude_files_regex = re.compile(exclude_files_regex, re.IGNORECASE)
+        files_to_scan = filter(
+            lambda file: not exclude_files_regex.search(file),
+            files_to_scan,
+        )
+
+    for file in sorted(files_to_scan):
+        output.scan_file(file)
+
+    return output
+
+
+def get_secrets_not_in_baseline(results, baseline):
+    """
+    :type results: SecretsCollection
+    :param results: SecretsCollection of current results
+
+    :type baseline: SecretsCollection
+    :param baseline: SecretsCollection of baseline results.
+                     This will be updated accordingly (by reference)
+
+    :rtype: SecretsCollection
+    :returns: SecretsCollection of new results (filtering out baseline)
+    """
+    exclude_files_regex = None
+    if baseline.exclude_files:
+        exclude_files_regex = re.compile(baseline.exclude_files, re.IGNORECASE)
+
+    new_secrets = SecretsCollection()
+    for filename in results.data:
+        if exclude_files_regex and exclude_files_regex.search(filename):
+            continue
+
+        if filename not in baseline.data:
+            # We don't have a previous record of this file, so obviously
+            # everything is new.
+            new_secrets.data[filename] = results.data[filename]
+            continue
+
+        # The __hash__ method of PotentialSecret makes this work
+        filtered_results = {
+            secret: secret
+            for secret in results.data[filename]
+            if secret not in baseline.data[filename]
+        }
+
+        if filtered_results:
+            new_secrets.data[filename] = filtered_results
+
+    return new_secrets
+
+
+def trim_baseline_of_removed_secrets(results, baseline, filelist):
+    """
+    NOTE: filelist is not a comprehensive list of all files in the repo
+    (because we can't be sure whether --all-files is passed in as a
+    parameter to pre-commit).
+
+    :type results: SecretsCollection
+    :type baseline: SecretsCollection
+
+    :type filelist: list(str)
+    :param filelist: filenames that are scanned.
+
+    :rtype: bool
+    :returns: True if baseline was updated
+    """
+    updated = False
+    for filename in filelist:
+        if filename not in baseline.data:
+            # Nothing to modify, because not even there in the first place.
+            continue
+
+        if filename not in results.data:
+            # All secrets relating to that file was removed.
+            # We know this because:
+            #   1. It's a file that was scanned (in filelist)
+            #   2. It was in the baseline
+            #   3. It has no results now.
+            del baseline.data[filename]
+            updated = True
+            continue
+
+        # We clone the baseline, so that we can modify the baseline,
+        # without messing up the iteration.
+        for baseline_secret in baseline.data[filename].copy():
+            new_secret_found = results.get_secret(
+                filename,
+                baseline_secret.secret_hash,
+                baseline_secret.type,
+            )
+
+            if not new_secret_found:
+                # No longer in results, so can remove from baseline
+                old_secret_to_delete = baseline.get_secret(
+                    filename,
+                    baseline_secret.secret_hash,
+                    baseline_secret.type,
+                )
+                del baseline.data[filename][old_secret_to_delete]
+                updated = True
+
+            elif new_secret_found.lineno != baseline_secret.lineno:
+                # Secret moved around, should update baseline with new location
+                old_secret_to_update = baseline.get_secret(
+                    filename,
+                    baseline_secret.secret_hash,
+                    baseline_secret.type,
+                )
+                old_secret_to_update.lineno = new_secret_found.lineno
+                updated = True
+
+    return updated
+
+
+def format_baseline_for_output(baseline):
+    """
+    :type baseline: dict
+    :rtype: str
+    """
+    lines = []
+    for filename, secret_list in baseline['results'].items():
+        lines_secrets = '\n'.join(
+            'Line %d: %s' % (x['line_number'], x['secret_value']) for x in secret_list
+        )
+        line = f"""
+Filename: {filename}
+{lines_secrets}
+"""
+        lines.append(line)
+
+    return '\n'.join(lines)
+
+
+def _get_git_tracked_files(rootdir='.'):
+    """Parsing .gitignore rules is hard.
+
+    However, a way we can get around this problem by just listing all
+    currently tracked git files, and start our search from there.
+    After all, if it isn't in the git repo, we're not concerned about
+    it, because secrets aren't being entered in a shared place.
+
+    :type rootdir: str
+    :param rootdir: root directory of where you want to list files from
+
+    :rtype: set|None
+    :returns: filepaths to files which git currently tracks (locally)
+    """
+    output = []
+    try:
+        with open(os.devnull, 'w') as fnull:
+            git_files = subprocess.check_output(
+                [
+                    'git',
+                    '-C',
+                    rootdir,
+                    'ls-files',
+                ],
+                stderr=fnull,
+            )
+        for filename in git_files.decode('utf-8').split():
+            relative_path = util.get_relative_path_if_in_cwd(rootdir, filename)
+            if relative_path:
+                output.append(relative_path)
+    except subprocess.CalledProcessError:
+        pass
+    return output
+
+
+def _get_files_recursively(rootdir):
+    """Sometimes, we want to use this tool with non-git repositories.
+    This function allows us to do so.
+    """
+    output = []
+    for root, _, files in os.walk(rootdir):
+        for filename in files:
+            relative_path = util.get_relative_path_if_in_cwd(root, filename)
+            if relative_path:
+                output.append(relative_path)
+    return output

simple_detect_secrets/core/bidirectional_iterator.py

@@ -0,0 +1,34 @@
+class BidirectionalIterator:
+    def __init__(self, collection):
+        self.collection = collection
+        self.index = -1  # Starts on -1, as index is increased _before_ getting result
+        self.step_back_once = False
+
+    def __next__(self):
+        if self.step_back_once:
+            self.index -= 1
+            self.step_back_once = False
+        else:
+            self.index += 1
+
+        if self.index < 0:
+            raise StopIteration
+
+        try:
+            result = self.collection[self.index]
+        except IndexError:
+            raise StopIteration
+
+        return result
+
+    def next(self):  # pragma: no cover
+        return self.__next__()
+
+    def step_back_on_next_iteration(self):
+        self.step_back_once = True
+
+    def can_step_back(self):
+        return self.index > 0
+
+    def __iter__(self):  # pragma: no cover
+        return self

simple_detect_secrets/core/code_snippet.py

@@ -0,0 +1,101 @@
+import itertools
+
+from .color import AnsiColor, colorize
+
+
+class CodeSnippetHighlighter:
+    def get_code_snippet(self, file_lines, line_number, lines_of_context=5):
+        """
+        :type file_lines: iterable of str
+        :param file_lines: an iterator of lines in the file
+
+        :type line_number: int
+        :param line_number: line which you want to focus on
+
+        :type lines_of_context: int
+        :param lines_of_context: how many lines to display around the line you want
+            to focus on.
+
+        :rtype: CodeSnippet
+        """
+        secret_line_index = line_number - 1
+        end_line = secret_line_index + lines_of_context + 1
+
+        if secret_line_index <= lines_of_context:
+            start_line = 0
+            index_of_secret_in_output = secret_line_index
+        else:
+            start_line = secret_line_index - lines_of_context
+            index_of_secret_in_output = lines_of_context
+
+        return CodeSnippet(
+            list(
+                itertools.islice(
+                    file_lines,
+                    start_line,
+                    end_line,
+                ),
+            ),
+            start_line,
+            index_of_secret_in_output,
+        )
+
+
+class CodeSnippet:
+    def __init__(self, snippet, start_line, target_index):
+        """
+        :type snippet: iterable and indexable of str
+        :param snippet: lines of code extracted from file
+
+        :type start_line: int
+        :param start_line: first line number in segment
+
+        :type target_index: int
+        :param target_index: index in snippet of target line
+        """
+        self.lines = snippet
+        self.start_line = start_line
+        self.target_index = target_index
+
+    @property
+    def target_line(self):
+        return self.lines[self.target_index]
+
+    @target_line.setter
+    def target_line(self, value):
+        self.lines[self.target_index] = value
+
+    def add_line_numbers(self):
+        for index, line in enumerate(self.lines):
+            self.lines[index] = f'{self.get_line_number(self.start_line + index + 1)}:{line}'
+
+        return self
+
+    def highlight_line(self, payload):
+        """
+        :type payload: str
+        :param payload: string to highlight, on chosen line
+        """
+        index_of_payload = self.target_line.lower().index(payload.lower())
+        end_of_payload = index_of_payload + len(payload)
+
+        self.target_line = f'{self.target_line[:index_of_payload]}{self.apply_highlight(self.target_line[index_of_payload:end_of_payload])}{self.target_line[end_of_payload:]}'
+
+        return self
+
+    def get_line_number(self, line_number):
+        """Broken out, for custom colorization."""
+        return colorize(
+            str(line_number),
+            AnsiColor.LIGHT_GREEN,
+        )
+
+    def apply_highlight(self, payload):
+        """Broken out, for custom colorization."""
+        return colorize(
+            payload,
+            AnsiColor.RED_BACKGROUND,
+        )
+
+    def __str__(self):
+        return '\n'.join(self.lines)

simple_detect_secrets/core/color.py

@@ -0,0 +1,14 @@
+from enum import Enum
+
+
+class AnsiColor(Enum):
+    RESET = '[0m'
+    BOLD = '[1m'
+    RED = '[91m'
+    RED_BACKGROUND = '[41m'
+    LIGHT_GREEN = '[92m'
+    PURPLE = '[95m'
+
+
+def colorize(text, color):
+    return f'\x1b{color.value}{text}\x1b{AnsiColor.RESET.value}'

simple_detect_secrets/core/common.py

@@ -0,0 +1,11 @@
+from .baseline import format_baseline_for_output
+
+
+def write_baseline_to_file(filename, data):
+    """
+    :type filename: str
+    :type data: dict
+    :rtype: None
+    """
+    with open(filename, 'w') as f:  # pragma: no cover
+        f.write(format_baseline_for_output(data) + '\n')

simple_detect_secrets/core/constants.py

@@ -0,0 +1,43 @@
+from enum import Enum
+
+# We don't scan files with these extensions.
+# NOTE: We might be able to do this better with
+#       `subprocess.check_output(['file', filename])`
+#       and look for "ASCII text", but that might be more expensive.
+#
+#       Definitely something to look into, if this list gets unruly long.
+IGNORED_FILE_EXTENSIONS = {
+    '.7z',
+    '.bmp',
+    '.bz2',
+    '.dmg',
+    '.eot',
+    '.exe',
+    '.gif',
+    '.gz',
+    '.ico',
+    '.jar',
+    '.jpg',
+    '.jpeg',
+    '.mo',
+    '.png',
+    '.rar',
+    '.realm',
+    '.s7z',
+    '.svg',
+    '.tar',
+    '.tif',
+    '.tiff',
+    '.ttf',
+    '.webp',
+    '.woff',
+    '.xls',
+    '.xlsx',
+    '.zip',
+}
+
+
+class VerifiedResult(Enum):
+    UNVERIFIED = 1
+    VERIFIED_FALSE = 2
+    VERIFIED_TRUE = 3

simple_detect_secrets/core/log.py

@@ -0,0 +1,51 @@
+import logging
+import sys
+
+
+def get_logger(name=None, format_string=None):
+    """
+    :type name: str
+    :param name: used for declaring log channels.
+
+    :type format_string: str
+    :param format_string: for custom formatting
+    """
+    logging.captureWarnings(True)
+    log = logging.getLogger(name)
+
+    # Bind custom method to instance.
+    # Source: https://stackoverflow.com/a/2982
+    log.set_debug_level = _set_debug_level.__get__(log)
+    log.set_debug_level(0)
+
+    if not format_string:
+        format_string = '[%(module)s]\t%(levelname)s\t%(message)s'
+
+    # Setting up log formats
+    log.handlers = []
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(
+        logging.Formatter(format_string),
+    )
+    log.addHandler(handler)
+
+    return log
+
+
+def _set_debug_level(self, debug_level):
+    """
+    :type debug_level: int, between 0-2
+    :param debug_level: configure verbosity of log
+    """
+    mapping = {
+        0: logging.ERROR,
+        1: logging.INFO,
+        2: logging.DEBUG,
+    }
+
+    self.setLevel(
+        mapping[min(debug_level, 2)],
+    )
+
+
+log = get_logger()

simple_detect_secrets/core/potential_secret.py

@@ -0,0 +1,92 @@
+class PotentialSecret:
+    """This custom data type represents a string found, matching the
+    plugin rules defined in SecretsCollection, that has the potential
+    to be a secret that we actually care about.
+
+    "Potential" is the operative word here, because of the nature of
+    false positives.
+
+    We use this custom class so that we can more easily generate data
+    structures and do object-based comparisons with other PotentialSecrets,
+    without actually knowing what the secret is.
+    """
+
+    def __init__(
+        self,
+        typ,
+        filename,
+        secret,
+        lineno=0,
+        is_secret=None,
+    ):
+        """
+        :type typ: str
+        :param typ: human-readable secret type, defined by the plugin
+                    that generated this PotentialSecret.
+                    e.g. "High Entropy String"
+
+        :type filename: str
+        :param filename: name of file that this secret was found
+
+        :type secret: str
+        :param secret: the actual secret identified
+
+        :type lineno: int
+        :param lineno: location of secret, within filename.
+                       Merely used as a reference for easy triage.
+
+        :type is_secret: bool|None
+        :param is_secret: whether or not the secret is a true- or false- positive
+
+        :type is_verified: bool
+        :param is_verified: whether the secret has been externally verified
+        """
+        self.type = typ
+        self.filename = filename
+        self.lineno = lineno
+        self.set_secret(secret)
+        self.is_secret = is_secret
+        self.is_verified = False
+
+        # If two PotentialSecrets have the same values for these fields,
+        # they are considered equal. Note that line numbers aren't included
+        # in this, because line numbers are subject to change.
+        self.fields_to_compare = ['filename', 'secret_value', 'type']
+
+    def set_secret(self, secret):
+        self.secret_value = secret
+
+    def json(self):
+        """Custom JSON encoder"""
+        attributes = {
+            'type': self.type,
+            'filename': self.filename,
+            'line_number': self.lineno,
+            'secret_value': self.secret_value,
+            'is_verified': self.is_verified,
+        }
+
+        if self.is_secret is not None:
+            attributes['is_secret'] = self.is_secret
+
+        return attributes
+
+    def __eq__(self, other):
+        return all(
+            getattr(self, field) == getattr(other, field) for field in self.fields_to_compare
+        )
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return hash(
+            tuple(getattr(self, x) for x in self.fields_to_compare),
+        )
+
+    def __str__(self):  # pragma: no cover
+        return ('Secret Type: %s\nLocation:    %s:%d\n') % (
+            self.type,
+            self.filename,
+            self.lineno,
+        )