credsweeper 1.11.5__py3-none-any.whl → 1.12.0__py3-none-any.whl

credsweeper/filters/value_dictionary_keyword_check.py

@@ -1,14 +1,16 @@
+from typing import Optional
+
 from credsweeper.common import static_keyword_checklist
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueDictionaryKeywordCheck(Filter):
     """Check that no word from dictionary present in the candidate value."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_discord_bot_check.py

@@ -1,17 +1,18 @@
 import contextlib
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 from credsweeper.filters.value_entropy_base64_check import ValueEntropyBase64Check
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 
 class ValueDiscordBotCheck(Filter):
     """Discord bot Token"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_entropy_base_check.py

@@ -1,16 +1,17 @@
 from abc import abstractmethod
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueEntropyBaseCheck(Filter):
     """Check that candidate value has minimal Shanon Entropy for appropriated base"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     @staticmethod

credsweeper/filters/value_file_path_check.py

@@ -1,10 +1,13 @@
+from typing import Optional
+
 from credsweeper.common import static_keyword_checklist
 from credsweeper.common.constants import Chars
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter, ValueEntropyBase64Check
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.filters.value_entropy_base64_check import ValueEntropyBase64Check
+from credsweeper.utils.util import Util
 
 
 class ValueFilePathCheck(Filter):
@@ -17,7 +20,7 @@ class ValueFilePathCheck(Filter):
     unusual_windows_symbols_in_path = "\t\n\r!$@`&*(){}<>+=;,~^"
     unusual_linux_symbols_in_path = "\t\n\r!@`&*<>+=;,~^:\\"
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_github_check.py

@@ -1,19 +1,20 @@
 import binascii
 import contextlib
+from typing import Optional
 
 import base62
 
 from credsweeper.common.constants import ASCII
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueGitHubCheck(Filter):
     """GitHub Classic Token validation"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_grafana_check.py

@@ -1,17 +1,18 @@
 import contextlib
 import json
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueGrafanaCheck(Filter):
     """Grafana Provisioned API Key and Access Policy Token"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_grafana_service_check.py

@@ -1,18 +1,19 @@
 import binascii
 import contextlib
 import struct
+from typing import Optional
 
 from credsweeper.common.constants import ASCII
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueGrafanaServiceCheck(Filter):
     """Check that candidate have a known structure"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_hex_number_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueHexNumberCheck(Filter):
@@ -11,7 +12,7 @@ class ValueHexNumberCheck(Filter):
 
     HEX_08_64_VALUE_REGEX = re.compile(r"^0x[0-9a-f]{1,16}$")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_jfrog_token_check.py

@@ -1,20 +1,21 @@
 import contextlib
 import re
+from typing import Optional
 
 import base58
 
 from credsweeper.common.constants import ASCII
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueJfrogTokenCheck(Filter):
     """Check that candidate have a known structure JFROG token"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         # reftkn:01:0123456789:abcdefGhijklmnoPqrstuVwxyz0
         self._pattern = re.compile(r"reftkn:\d+:\d+:[\w_/+-]+")
 

credsweeper/filters/value_json_web_key_check.py

@@ -1,10 +1,11 @@
 import contextlib
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueJsonWebKeyCheck(Filter):
@@ -15,7 +16,7 @@ class ValueJsonWebKeyCheck(Filter):
     https://datatracker.ietf.org/doc/html/rfc7518
     """
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_json_web_token_check.py

@@ -1,11 +1,12 @@
 import contextlib
 import json
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueJsonWebTokenCheck(Filter):
@@ -24,7 +25,7 @@ class ValueJsonWebTokenCheck(Filter):
         "ext", "crit", "keys", "id", "role", "token", "secret", "password", "nonce"
     }
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_last_word_check.py

@@ -1,13 +1,15 @@
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from typing import Optional
+
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueLastWordCheck(Filter):
     """Check that secret is not short value that ends with `:`."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/{value_dictionary_value_length_check.py → value_length_check.py}

@@ -1,13 +1,19 @@
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from typing import Optional
+
+from credsweeper.common.constants import MIN_VALUE_LENGTH, MAX_LINE_LENGTH
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
-class ValueDictionaryValueLengthCheck(Filter):
-    """Check that candidate length is between 5 and 30."""
+class ValueLengthCheck(Filter):
+    """Check that candidate value length is between MIN and MAX."""
 
-    def __init__(self, config: Config = None, min_len: int = 4, max_len: int = 31) -> None:
+    def __init__(self,
+                 config: Optional[Config] = None,
+                 min_len: int = MIN_VALUE_LENGTH,
+                 max_len: int = MAX_LINE_LENGTH) -> None:
         self.min_len = min_len
         self.max_len = max_len
 

credsweeper/filters/value_method_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueMethodCheck(Filter):
@@ -14,7 +15,7 @@ class ValueMethodCheck(Filter):
 
     PATTERN = re.compile(r"^[~.\->:0-9A-Za-z_]+\(.*\)")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_not_allowed_pattern_check.py

@@ -1,10 +1,11 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueNotAllowedPatternCheck(Filter):
@@ -15,7 +16,7 @@ class ValueNotAllowedPatternCheck(Filter):
         f"{Util.get_regex_combine_or(NOT_ALLOWED)}$",  #
         flags=re.IGNORECASE)
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_not_part_encoded_check.py

@@ -2,10 +2,10 @@ import re
 from typing import Optional
 
 from credsweeper.common import static_keyword_checklist
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueNotPartEncodedCheck(Filter):
@@ -16,7 +16,7 @@ class ValueNotPartEncodedCheck(Filter):
     BASE64_ENCODED_DATA_PATTERN_AFTER = re.compile(
         r"(^|[^A-Za-z0-9]+)(?P<val>(([A-Za-z0-9=_-]{4}){4,64})|(([A-Za-z0-9=+/]{4}){4,64}))([^=A-Za-z0-9]+|$)")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     @staticmethod

credsweeper/filters/value_number_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueNumberCheck(Filter):
@@ -12,7 +13,7 @@ class ValueNumberCheck(Filter):
     HEX_VALUE_REGEX = re.compile("^(0x)?[0-9a-f]{1,128}[ul]{0,3}$")
     DEC_VALUE_REGEX = re.compile("^-?[0-9]{1,20}[ul]{0,3}$")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_pattern_check.py

@@ -1,10 +1,11 @@
 import re
+from typing import Optional
 
-from credsweeper.common.constants import DEFAULT_PATTERN_LEN
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.common.constants import DEFAULT_PATTERN_LEN, MAX_LINE_LENGTH
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValuePatternCheck(Filter):
@@ -22,36 +23,60 @@ class ValuePatternCheck(Filter):
     Default pattern LEN is 4
     """
 
-    def __init__(self, config: Config = None, pattern_len: int = DEFAULT_PATTERN_LEN):
+    MAX_PATTERN_LENGTH = int(MAX_LINE_LENGTH).bit_length()
+
+    def __init__(self, config: Optional[Config] = None, pattern_len: Optional[int] = None):
         """Create ValuePatternCheck with a specific pattern_len to check.
 
         Args:
             config: pattern len to use during check. DEFAULT_PATTERN_LEN by default
+            pattern_len: size of constant pattern length for any value size or None for dynamic pattern size
 
         """
-        self.pattern_len = pattern_len
-        # use non whitespace symbol pattern
-        self.pattern = re.compile(fr"(\S)\1{{{str(self.pattern_len - 1)},}}")
+        patterns_count = 1 + ValuePatternCheck.MAX_PATTERN_LENGTH
+        if pattern_len is None:
+            self.pattern_len = -1
+            # pattern length depends on value length
+            self.pattern_lengths = [max(x, DEFAULT_PATTERN_LEN) for x in range(patterns_count)]
+            self.patterns = [ValuePatternCheck.get_pattern(x) for x in range(patterns_count)]
+        elif isinstance(pattern_len, int) and DEFAULT_PATTERN_LEN <= pattern_len:
+            self.pattern_len = pattern_len
+            # constant pattern for any value length
+            self.pattern_lengths = [pattern_len] * patterns_count
+            self.patterns = [ValuePatternCheck.get_pattern(pattern_len)] * patterns_count
+        else:
+            raise ValueError(f"Wrong type of pattern length {type(pattern_len)} = {repr(pattern_len)}")
+
+    @staticmethod
+    def get_pattern(pattern_len: int) -> re.Pattern:
+        """Creates regex pattern to find N or more identical characters in sequence"""
+        if DEFAULT_PATTERN_LEN < pattern_len:
+            pattern = fr"(\S)\1{{{str(pattern_len - 1)},}}"
+        else:
+            pattern = r"(\S)\1{3,}"
+        return re.compile(pattern)
 
-    def equal_pattern_check(self, value: str) -> bool:
+    def equal_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value contain 4 and more same chars or numbers sequences.
 
         Args:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             True if contain and False if not
 
         """
-        if self.pattern.findall(value):
+        if self.patterns[bit_length].findall(value):
             return True
         return False
 
-    def ascending_pattern_check(self, value: str) -> bool:
+    def ascending_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value contain 4 and more ascending chars or numbers sequences.
 
         Arg:
             value: credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             True if contain and False if not
@@ -64,15 +89,16 @@ class ValuePatternCheck(Filter):
             else:
                 count = 1
                 continue
-            if count == self.pattern_len:
+            if count == self.pattern_lengths[bit_length]:
                 return True
         return False
 
-    def descending_pattern_check(self, value: str) -> bool:
+    def descending_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value contain 4 and more descending chars or numbers sequences.
 
         Arg:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             boolean variable. True if contain and False if not
@@ -85,59 +111,44 @@ class ValuePatternCheck(Filter):
             else:
                 count = 1
                 continue
-            if count == self.pattern_len:
+            if count == self.pattern_lengths[bit_length]:
                 return True
         return False
 
-    def check_val(self, value: str) -> bool:
+    def check_val(self, value: str, bit_length: int) -> bool:
         """Cumulative value check.
 
         Arg:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             boolean variable. True if contain and False if not
 
         """
-        if self.equal_pattern_check(value):
+        if self.equal_pattern_check(value, bit_length):
             return True
-        if self.ascending_pattern_check(value):
+        if self.ascending_pattern_check(value, bit_length):
             return True
-        if self.descending_pattern_check(value):
+        if self.descending_pattern_check(value, bit_length):
             return True
         return False
 
-    def duple_pattern_check(self, value: str) -> bool:
+    def duple_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value is a duplet value with possible patterns.
 
         Arg:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             boolean variable. True if contain and False if not
 
         """
-        # 001122334455... case
-        pair_duple = True
-        # 0102030405... case
-        even_duple = True
-        even_prev = value[0]
         even_value = value[0::2]
-        # 1020304050... case
-        odd_duple = True
-        odd_prev = value[1]
         odd_value = value[1::2]
-        for even_i, odd_i in zip(even_value, odd_value):
-            pair_duple &= even_i == odd_i
-            even_duple &= even_i == even_prev
-            odd_duple &= odd_i == odd_prev
-            if not pair_duple and not even_duple and not odd_duple:
-                break
-        else:
-            if pair_duple or odd_duple:
-                return self.check_val(even_value)
-            if even_duple:
-                return self.check_val(odd_value)
+        if self.check_val(even_value, bit_length) and self.check_val(odd_value, bit_length):
+            return True
         return False
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
@@ -151,13 +162,22 @@ class ValuePatternCheck(Filter):
             boolean variable. True, if need to filter candidate and False if left
 
         """
-        if len(line_data.value) < self.pattern_len:
+        value_length = len(line_data.value)
+        bit_length = max(DEFAULT_PATTERN_LEN, value_length.bit_length())
+
+        if ValuePatternCheck.MAX_PATTERN_LENGTH < bit_length:
+            # huge values may contain anything
+            return False
+
+        if 0 <= value_length < self.pattern_len or value_length < self.pattern_lengths[bit_length]:
+            # too short value
             return True
 
-        if self.check_val(line_data.value):
+        if self.check_val(line_data.value, bit_length):
             return True
 
-        if 2 * self.pattern_len <= len(line_data.value) and self.duple_pattern_check(line_data.value):
+        if 2 * self.pattern_lengths[bit_length] <= value_length \
+                and self.duple_pattern_check(line_data.value, bit_length):
             return True
 
         return False

credsweeper/filters/value_similarity_check.py

@@ -1,13 +1,15 @@
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from typing import Optional
+
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueSimilarityCheck(Filter):
     """Check if candidate value is at least 70% same as candidate keyword. Like: `secret = "mysecret"`."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_split_keyword_check.py

@@ -1,16 +1,17 @@
+from typing import Optional
 from typing import Union
 
 from credsweeper.common import static_keyword_checklist
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueSplitKeywordCheck(Filter):
     """Check value by splitting with standard whitespace separators and any word is not matched in checklist."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool: