credsweeper 1.11.5__py3-none-any.whl → 1.13.3__py3-none-any.whl

credsweeper/filters/value_blocklist_check.py

@@ -1,7 +1,9 @@
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from typing import Optional
+
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueBlocklistCheck(Filter):
@@ -18,7 +20,7 @@ class ValueBlocklistCheck(Filter):
         "undefined",
     ]
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_camel_case_check.py

@@ -1,20 +1,21 @@
 import re
+from typing import Optional
 
 from credsweeper.common import static_keyword_checklist
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueCamelCaseCheck(Filter):
     """Check that candidate is not written in camel case."""
 
-    CAMEL_CASE = ["^([a-z]+([A-Z][a-z]+)+)$", "^([A-Z][a-z]+([A-Z][a-z]+)+)$"]
+    CAMEL_CASE = ["[a-z]+([A-Z][a-z]+)+", "[A-Z][a-z]+([A-Z][a-z]+)+"]
     CAMEL_CASE_PATTERN = re.compile(Util.get_regex_combine_or(CAMEL_CASE))
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
@@ -30,7 +31,7 @@ class ValueCamelCaseCheck(Filter):
         """
         if line_data.is_well_quoted_value:
             return False
-        if self.CAMEL_CASE_PATTERN.match(line_data.value):
+        if self.CAMEL_CASE_PATTERN.fullmatch(line_data.value):
             return static_keyword_checklist.check_morphemes(line_data.value.lower(), 1)
 
         return False

credsweeper/filters/value_dictionary_keyword_check.py

@@ -1,14 +1,16 @@
+from typing import Optional
+
 from credsweeper.common import static_keyword_checklist
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueDictionaryKeywordCheck(Filter):
     """Check that no word from dictionary present in the candidate value."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_discord_bot_check.py

@@ -1,17 +1,18 @@
 import contextlib
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 from credsweeper.filters.value_entropy_base64_check import ValueEntropyBase64Check
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 
 class ValueDiscordBotCheck(Filter):
     """Discord bot Token"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_entropy_base_check.py

@@ -1,16 +1,17 @@
 from abc import abstractmethod
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueEntropyBaseCheck(Filter):
     """Check that candidate value has minimal Shanon Entropy for appropriated base"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     @staticmethod

credsweeper/filters/value_file_path_check.py

@@ -1,10 +1,13 @@
+from typing import Optional
+
 from credsweeper.common import static_keyword_checklist
 from credsweeper.common.constants import Chars
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter, ValueEntropyBase64Check
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.filters.value_entropy_base64_check import ValueEntropyBase64Check
+from credsweeper.utils.util import Util
 
 
 class ValueFilePathCheck(Filter):
@@ -17,7 +20,7 @@ class ValueFilePathCheck(Filter):
     unusual_windows_symbols_in_path = "\t\n\r!$@`&*(){}<>+=;,~^"
     unusual_linux_symbols_in_path = "\t\n\r!@`&*<>+=;,~^:\\"
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
@@ -32,6 +35,8 @@ class ValueFilePathCheck(Filter):
 
         """
         value = line_data.value
+        bit_length = len(value).bit_length()
+        morpheme_threshold = 1 if 6 > bit_length else bit_length - 4
         contains_unix_separator = '/' in value
         if contains_unix_separator:
             if ("://" in value  #
@@ -42,14 +47,14 @@ class ValueFilePathCheck(Filter):
                     or value.startswith("//") and ':' == line_data.separator):
                 # common case for url definition or aliases
                 # or _keyword_://example.com where : is the separator
-                return static_keyword_checklist.check_morphemes(value.lower(), 1)
+                return static_keyword_checklist.check_morphemes(value.lower(), morpheme_threshold)
             # base64 encoded data might look like linux path
             min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(value))
             # get minimal entropy to compare with shannon entropy of found value
             # min_entropy == 0 means that the value cannot be checked with the entropy due high variance
             for i in value:
                 if i not in self.base64stdpad_possible_set:
-                    # value contains wrong BASE64STDPAD_CHARS symbols like -_
+                    # value contains wrong BASE64STDPAD_CHARS symbols like -_.
                     break
             else:
                 # all symbols are from base64 alphabet
@@ -71,5 +76,5 @@ class ValueFilePathCheck(Filter):
                     break
             else:
                 if contains_unix_separator ^ contains_windows_separator:
-                    return static_keyword_checklist.check_morphemes(value.lower(), 1)
+                    return static_keyword_checklist.check_morphemes(value.lower(), morpheme_threshold)
         return False

credsweeper/filters/value_github_check.py

@@ -1,19 +1,20 @@
 import binascii
 import contextlib
+from typing import Optional
 
 import base62
 
 from credsweeper.common.constants import ASCII
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueGitHubCheck(Filter):
-    """GitHub Classic Token validation"""
+    """NPM or GitHub Classic Token validation"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
@@ -28,8 +29,9 @@ class ValueGitHubCheck(Filter):
 
         """
         # https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
+        # https://github.blog/security/announcing-npms-new-access-token-format/
         with contextlib.suppress(Exception):
-            if line_data.value.startswith("gh") and '_' == line_data.value[3]:
+            if (line_data.value.startswith("gh") and '_' == line_data.value[3]) or line_data.value.startswith("npm_"):
                 token = line_data.value[4:-6]
                 data = token.encode(ASCII, errors="strict")
                 crc32sum = binascii.crc32(data)

credsweeper/filters/value_grafana_check.py

@@ -1,17 +1,18 @@
 import contextlib
 import json
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueGrafanaCheck(Filter):
     """Grafana Provisioned API Key and Access Policy Token"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_grafana_service_check.py

@@ -1,18 +1,19 @@
 import binascii
 import contextlib
 import struct
+from typing import Optional
 
 from credsweeper.common.constants import ASCII
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueGrafanaServiceCheck(Filter):
     """Check that candidate have a known structure"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_hex_number_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueHexNumberCheck(Filter):
@@ -11,7 +12,7 @@ class ValueHexNumberCheck(Filter):
 
     HEX_08_64_VALUE_REGEX = re.compile(r"^0x[0-9a-f]{1,16}$")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_jfrog_token_check.py

@@ -1,20 +1,21 @@
 import contextlib
 import re
+from typing import Optional
 
 import base58
 
 from credsweeper.common.constants import ASCII
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueJfrogTokenCheck(Filter):
     """Check that candidate have a known structure JFROG token"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         # reftkn:01:0123456789:abcdefGhijklmnoPqrstuVwxyz0
         self._pattern = re.compile(r"reftkn:\d+:\d+:[\w_/+-]+")
 

credsweeper/filters/value_json_web_key_check.py

@@ -1,10 +1,11 @@
 import contextlib
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueJsonWebKeyCheck(Filter):
@@ -15,7 +16,7 @@ class ValueJsonWebKeyCheck(Filter):
     https://datatracker.ietf.org/doc/html/rfc7518
     """
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_json_web_token_check.py

@@ -1,11 +1,12 @@
 import contextlib
 import json
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueJsonWebTokenCheck(Filter):
@@ -24,7 +25,7 @@ class ValueJsonWebTokenCheck(Filter):
         "ext", "crit", "keys", "id", "role", "token", "secret", "password", "nonce"
     }
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_last_word_check.py

@@ -1,13 +1,15 @@
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from typing import Optional
+
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueLastWordCheck(Filter):
     """Check that secret is not short value that ends with `:`."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/{value_dictionary_value_length_check.py → value_length_check.py}

@@ -1,13 +1,19 @@
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from typing import Optional
+
+from credsweeper.common.constants import MIN_VALUE_LENGTH, MAX_LINE_LENGTH
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
-class ValueDictionaryValueLengthCheck(Filter):
-    """Check that candidate length is between 5 and 30."""
+class ValueLengthCheck(Filter):
+    """Check that candidate value length is between MIN and MAX."""
 
-    def __init__(self, config: Config = None, min_len: int = 4, max_len: int = 31) -> None:
+    def __init__(self,
+                 config: Optional[Config] = None,
+                 min_len: int = MIN_VALUE_LENGTH,
+                 max_len: int = MAX_LINE_LENGTH) -> None:
         self.min_len = min_len
         self.max_len = max_len
 

credsweeper/filters/value_method_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueMethodCheck(Filter):
@@ -14,7 +15,7 @@ class ValueMethodCheck(Filter):
 
     PATTERN = re.compile(r"^[~.\->:0-9A-Za-z_]+\(.*\)")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_morphemes_check.py

@@ -0,0 +1,43 @@
+from typing import Optional
+
+from credsweeper.common import static_keyword_checklist
+from credsweeper.common.constants import MAX_LINE_LENGTH
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
+from credsweeper.file_handler.analysis_target import AnalysisTarget
+from credsweeper.filters.filter import Filter
+
+
+class ValueMorphemesCheck(Filter):
+    """Check value for a threshold of morphemes count"""
+
+    THRESHOLDS_X3 = int(MAX_LINE_LENGTH).bit_length()
+    # one morpheme is very likely to be random generated even for 3 symbols
+    MAX_MORPHEMES_LIMIT = max(1, THRESHOLDS_X3 - 4)
+
+    def __init__(self, config: Optional[Config] = None, threshold: Optional[int] = None) -> None:
+        # threshold - minimum morphemes number in a value
+        if threshold is None:
+            # use dynamic thresholds
+            self.thresholds = [max(1, x - 4) for x in range(ValueMorphemesCheck.THRESHOLDS_X3)]
+        elif isinstance(threshold, int) and 0 <= threshold:
+            # constant thresholds for any pattern
+            self.thresholds = [threshold] * ValueMorphemesCheck.THRESHOLDS_X3
+        else:
+            raise ValueError(f"Wrong type of pattern length {type(threshold)} = {repr(threshold)}")
+
+    def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
+        """Run filter checks on received credential candidate data 'line_data'.
+
+        Args:
+            line_data: credential candidate data
+            target: multiline target from which line data was obtained
+
+        Return:
+            True, if need to filter candidate and False if left
+
+        """
+        threshold_id = len(line_data.value).bit_length()
+        # use the last (max) threshold in very huge value
+        threshold = self.thresholds[threshold_id] if len(self.thresholds) > threshold_id else self.thresholds[-1]
+        return static_keyword_checklist.check_morphemes(line_data.value.lower(), threshold)

credsweeper/filters/value_not_allowed_pattern_check.py

@@ -1,10 +1,11 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
 
 
 class ValueNotAllowedPatternCheck(Filter):
@@ -15,7 +16,7 @@ class ValueNotAllowedPatternCheck(Filter):
         f"{Util.get_regex_combine_or(NOT_ALLOWED)}$",  #
         flags=re.IGNORECASE)
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_not_part_encoded_check.py

@@ -2,10 +2,10 @@ import re
 from typing import Optional
 
 from credsweeper.common import static_keyword_checklist
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueNotPartEncodedCheck(Filter):
@@ -16,7 +16,7 @@ class ValueNotPartEncodedCheck(Filter):
     BASE64_ENCODED_DATA_PATTERN_AFTER = re.compile(
         r"(^|[^A-Za-z0-9]+)(?P<val>(([A-Za-z0-9=_-]{4}){4,64})|(([A-Za-z0-9=+/]{4}){4,64}))([^=A-Za-z0-9]+|$)")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     @staticmethod

credsweeper/filters/value_number_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueNumberCheck(Filter):
@@ -12,7 +13,7 @@ class ValueNumberCheck(Filter):
     HEX_VALUE_REGEX = re.compile("^(0x)?[0-9a-f]{1,128}[ul]{0,3}$")
     DEC_VALUE_REGEX = re.compile("^-?[0-9]{1,20}[ul]{0,3}$")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool: