credsweeper 1.11.6__py3-none-any.whl → 1.12.0__py3-none-any.whl

credsweeper/filters/value_file_path_check.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from credsweeper.common import static_keyword_checklist
 from credsweeper.common.constants import Chars
 from credsweeper.config.config import Config
@@ -18,7 +20,7 @@ class ValueFilePathCheck(Filter):
     unusual_windows_symbols_in_path = "\t\n\r!$@`&*(){}<>+=;,~^"
     unusual_linux_symbols_in_path = "\t\n\r!@`&*<>+=;,~^:\\"
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_github_check.py

@@ -1,5 +1,6 @@
 import binascii
 import contextlib
+from typing import Optional
 
 import base62
 
@@ -13,7 +14,7 @@ from credsweeper.filters.filter import Filter
 class ValueGitHubCheck(Filter):
     """GitHub Classic Token validation"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_grafana_check.py

@@ -1,5 +1,6 @@
 import contextlib
 import json
+from typing import Optional
 
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
@@ -11,7 +12,7 @@ from credsweeper.utils.util import Util
 class ValueGrafanaCheck(Filter):
     """Grafana Provisioned API Key and Access Policy Token"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_grafana_service_check.py

@@ -1,6 +1,7 @@
 import binascii
 import contextlib
 import struct
+from typing import Optional
 
 from credsweeper.common.constants import ASCII
 from credsweeper.config.config import Config
@@ -12,7 +13,7 @@ from credsweeper.filters.filter import Filter
 class ValueGrafanaServiceCheck(Filter):
     """Check that candidate have a known structure"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_hex_number_check.py

@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
@@ -11,7 +12,7 @@ class ValueHexNumberCheck(Filter):
 
     HEX_08_64_VALUE_REGEX = re.compile(r"^0x[0-9a-f]{1,16}$")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_jfrog_token_check.py

@@ -1,5 +1,6 @@
 import contextlib
 import re
+from typing import Optional
 
 import base58
 
@@ -14,7 +15,7 @@ from credsweeper.utils.util import Util
 class ValueJfrogTokenCheck(Filter):
     """Check that candidate have a known structure JFROG token"""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         # reftkn:01:0123456789:abcdefGhijklmnoPqrstuVwxyz0
         self._pattern = re.compile(r"reftkn:\d+:\d+:[\w_/+-]+")
 

credsweeper/filters/value_json_web_key_check.py

@@ -1,4 +1,5 @@
 import contextlib
+from typing import Optional
 
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
@@ -15,7 +16,7 @@ class ValueJsonWebKeyCheck(Filter):
     https://datatracker.ietf.org/doc/html/rfc7518
     """
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_json_web_token_check.py

@@ -1,5 +1,6 @@
 import contextlib
 import json
+from typing import Optional
 
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
@@ -24,7 +25,7 @@ class ValueJsonWebTokenCheck(Filter):
         "ext", "crit", "keys", "id", "role", "token", "secret", "password", "nonce"
     }
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_last_word_check.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
@@ -7,7 +9,7 @@ from credsweeper.filters.filter import Filter
 class ValueLastWordCheck(Filter):
     """Check that secret is not short value that ends with `:`."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/{value_dictionary_value_length_check.py → value_length_check.py}

@@ -1,13 +1,19 @@
+from typing import Optional
+
+from credsweeper.common.constants import MIN_VALUE_LENGTH, MAX_LINE_LENGTH
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.filters.filter import Filter
 
 
-class ValueDictionaryValueLengthCheck(Filter):
-    """Check that candidate length is between 5 and 30."""
+class ValueLengthCheck(Filter):
+    """Check that candidate value length is between MIN and MAX."""
 
-    def __init__(self, config: Config = None, min_len: int = 4, max_len: int = 31) -> None:
+    def __init__(self,
+                 config: Optional[Config] = None,
+                 min_len: int = MIN_VALUE_LENGTH,
+                 max_len: int = MAX_LINE_LENGTH) -> None:
         self.min_len = min_len
         self.max_len = max_len
 

credsweeper/filters/value_method_check.py

@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
@@ -14,7 +15,7 @@ class ValueMethodCheck(Filter):
 
     PATTERN = re.compile(r"^[~.\->:0-9A-Za-z_]+\(.*\)")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_not_allowed_pattern_check.py

@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
@@ -15,7 +16,7 @@ class ValueNotAllowedPatternCheck(Filter):
         f"{Util.get_regex_combine_or(NOT_ALLOWED)}$",  #
         flags=re.IGNORECASE)
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_not_part_encoded_check.py

@@ -16,7 +16,7 @@ class ValueNotPartEncodedCheck(Filter):
     BASE64_ENCODED_DATA_PATTERN_AFTER = re.compile(
         r"(^|[^A-Za-z0-9]+)(?P<val>(([A-Za-z0-9=_-]{4}){4,64})|(([A-Za-z0-9=+/]{4}){4,64}))([^=A-Za-z0-9]+|$)")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     @staticmethod

credsweeper/filters/value_number_check.py

@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
@@ -12,7 +13,7 @@ class ValueNumberCheck(Filter):
     HEX_VALUE_REGEX = re.compile("^(0x)?[0-9a-f]{1,128}[ul]{0,3}$")
     DEC_VALUE_REGEX = re.compile("^-?[0-9]{1,20}[ul]{0,3}$")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_pattern_check.py

@@ -1,6 +1,7 @@
 import re
+from typing import Optional
 
-from credsweeper.common.constants import DEFAULT_PATTERN_LEN
+from credsweeper.common.constants import DEFAULT_PATTERN_LEN, MAX_LINE_LENGTH
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
@@ -22,36 +23,60 @@ class ValuePatternCheck(Filter):
     Default pattern LEN is 4
     """
 
-    def __init__(self, config: Config = None, pattern_len: int = DEFAULT_PATTERN_LEN):
+    MAX_PATTERN_LENGTH = int(MAX_LINE_LENGTH).bit_length()
+
+    def __init__(self, config: Optional[Config] = None, pattern_len: Optional[int] = None):
         """Create ValuePatternCheck with a specific pattern_len to check.
 
         Args:
             config: pattern len to use during check. DEFAULT_PATTERN_LEN by default
+            pattern_len: size of constant pattern length for any value size or None for dynamic pattern size
 
         """
-        self.pattern_len = pattern_len
-        # use non whitespace symbol pattern
-        self.pattern = re.compile(fr"(\S)\1{{{str(self.pattern_len - 1)},}}")
+        patterns_count = 1 + ValuePatternCheck.MAX_PATTERN_LENGTH
+        if pattern_len is None:
+            self.pattern_len = -1
+            # pattern length depends on value length
+            self.pattern_lengths = [max(x, DEFAULT_PATTERN_LEN) for x in range(patterns_count)]
+            self.patterns = [ValuePatternCheck.get_pattern(x) for x in range(patterns_count)]
+        elif isinstance(pattern_len, int) and DEFAULT_PATTERN_LEN <= pattern_len:
+            self.pattern_len = pattern_len
+            # constant pattern for any value length
+            self.pattern_lengths = [pattern_len] * patterns_count
+            self.patterns = [ValuePatternCheck.get_pattern(pattern_len)] * patterns_count
+        else:
+            raise ValueError(f"Wrong type of pattern length {type(pattern_len)} = {repr(pattern_len)}")
+
+    @staticmethod
+    def get_pattern(pattern_len: int) -> re.Pattern:
+        """Creates regex pattern to find N or more identical characters in sequence"""
+        if DEFAULT_PATTERN_LEN < pattern_len:
+            pattern = fr"(\S)\1{{{str(pattern_len - 1)},}}"
+        else:
+            pattern = r"(\S)\1{3,}"
+        return re.compile(pattern)
 
-    def equal_pattern_check(self, value: str) -> bool:
+    def equal_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value contain 4 and more same chars or numbers sequences.
 
         Args:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             True if contain and False if not
 
         """
-        if self.pattern.findall(value):
+        if self.patterns[bit_length].findall(value):
             return True
         return False
 
-    def ascending_pattern_check(self, value: str) -> bool:
+    def ascending_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value contain 4 and more ascending chars or numbers sequences.
 
         Arg:
             value: credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             True if contain and False if not
@@ -64,15 +89,16 @@ class ValuePatternCheck(Filter):
             else:
                 count = 1
                 continue
-            if count == self.pattern_len:
+            if count == self.pattern_lengths[bit_length]:
                 return True
         return False
 
-    def descending_pattern_check(self, value: str) -> bool:
+    def descending_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value contain 4 and more descending chars or numbers sequences.
 
         Arg:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             boolean variable. True if contain and False if not
@@ -85,59 +111,44 @@ class ValuePatternCheck(Filter):
             else:
                 count = 1
                 continue
-            if count == self.pattern_len:
+            if count == self.pattern_lengths[bit_length]:
                 return True
         return False
 
-    def check_val(self, value: str) -> bool:
+    def check_val(self, value: str, bit_length: int) -> bool:
         """Cumulative value check.
 
         Arg:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             boolean variable. True if contain and False if not
 
         """
-        if self.equal_pattern_check(value):
+        if self.equal_pattern_check(value, bit_length):
             return True
-        if self.ascending_pattern_check(value):
+        if self.ascending_pattern_check(value, bit_length):
             return True
-        if self.descending_pattern_check(value):
+        if self.descending_pattern_check(value, bit_length):
             return True
         return False
 
-    def duple_pattern_check(self, value: str) -> bool:
+    def duple_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value is a duplet value with possible patterns.
 
         Arg:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             boolean variable. True if contain and False if not
 
         """
-        # 001122334455... case
-        pair_duple = True
-        # 0102030405... case
-        even_duple = True
-        even_prev = value[0]
         even_value = value[0::2]
-        # 1020304050... case
-        odd_duple = True
-        odd_prev = value[1]
         odd_value = value[1::2]
-        for even_i, odd_i in zip(even_value, odd_value):
-            pair_duple &= even_i == odd_i
-            even_duple &= even_i == even_prev
-            odd_duple &= odd_i == odd_prev
-            if not pair_duple and not even_duple and not odd_duple:
-                break
-        else:
-            if pair_duple or odd_duple:
-                return self.check_val(even_value)
-            if even_duple:
-                return self.check_val(odd_value)
+        if self.check_val(even_value, bit_length) and self.check_val(odd_value, bit_length):
+            return True
         return False
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
@@ -151,13 +162,22 @@ class ValuePatternCheck(Filter):
             boolean variable. True, if need to filter candidate and False if left
 
         """
-        if len(line_data.value) < self.pattern_len:
+        value_length = len(line_data.value)
+        bit_length = max(DEFAULT_PATTERN_LEN, value_length.bit_length())
+
+        if ValuePatternCheck.MAX_PATTERN_LENGTH < bit_length:
+            # huge values may contain anything
+            return False
+
+        if 0 <= value_length < self.pattern_len or value_length < self.pattern_lengths[bit_length]:
+            # too short value
             return True
 
-        if self.check_val(line_data.value):
+        if self.check_val(line_data.value, bit_length):
             return True
 
-        if 2 * self.pattern_len <= len(line_data.value) and self.duple_pattern_check(line_data.value):
+        if 2 * self.pattern_lengths[bit_length] <= value_length \
+                and self.duple_pattern_check(line_data.value, bit_length):
             return True
 
         return False

credsweeper/filters/value_similarity_check.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
@@ -7,7 +9,7 @@ from credsweeper.filters.filter import Filter
 class ValueSimilarityCheck(Filter):
     """Check if candidate value is at least 70% same as candidate keyword. Like: `secret = "mysecret"`."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_split_keyword_check.py

@@ -1,3 +1,4 @@
+from typing import Optional
 from typing import Union
 
 from credsweeper.common import static_keyword_checklist
@@ -10,7 +11,7 @@ from credsweeper.filters.filter import Filter
 class ValueSplitKeywordCheck(Filter):
     """Check value by splitting with standard whitespace separators and any word is not matched in checklist."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_string_type_check.py

@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
@@ -23,10 +24,11 @@ class ValueStringTypeCheck(Filter):
     False otherwise
     """
 
-    MULTIBYTE_PATTERN = re.compile(r"(\s*(0x)?[0-9a-f]{1,3}\s*,){8,80}", flags=re.IGNORECASE)
+    MULTIBYTE_PATTERN = re.compile(r"((0x)?[0-9a-f]{1,16}[UL]*)(\s*,\s*((0x)?[0-9a-f]{1,16}[UL]*)){3}",
+                                   flags=re.IGNORECASE)
 
-    def __init__(self, config: Config) -> None:
-        self.check_for_literals = config.check_for_literals
+    def __init__(self, config: Optional[Config] = None, check_for_literals=True) -> None:
+        self.check_for_literals = check_for_literals
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
         """Run filter checks on received credential candidate data 'line_data'.
@@ -42,7 +44,7 @@ class ValueStringTypeCheck(Filter):
         if not self.check_for_literals or line_data.url_part:
             return False
 
-        if ValueStringTypeCheck.MULTIBYTE_PATTERN.match(line_data.value):
+        if ValueStringTypeCheck.MULTIBYTE_PATTERN.search(line_data.value):
             return False
 
         if line_data.is_source_file_with_quotes() \

credsweeper/filters/value_token_base_check.py

@@ -1,5 +1,6 @@
 import contextlib
 from abc import abstractmethod
+from typing import Optional
 from typing import Tuple
 
 from credsweeper.config.config import Config
@@ -26,7 +27,7 @@ class ValueTokenBaseCheck(Filter):
         64: 2.15981241,
     }
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         self.__hop_stat = HopStat()
 
     @staticmethod

credsweeper/filters/value_token_check.py

@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 
 from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
@@ -17,9 +18,9 @@ class ValueTokenCheck(Filter):
 
     """
 
-    SPLIT_PATTERN = r"(?<!,) (?!,)|;|\)|\(|{|}|<|>|\[|\]|`"
+    SPLIT_PATTERN = re.compile(r"(?<!\W) (?!\W)|[;(){}<>[\]`]")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/ml_model/features/__init__.py

@@ -5,6 +5,7 @@ from credsweeper.ml_model.features.is_secret_numeric import IsSecretNumeric
 from credsweeper.ml_model.features.length_of_attribute import LengthOfAttribute
 from credsweeper.ml_model.features.morpheme_dense import MorphemeDense
 from credsweeper.ml_model.features.rule_name import RuleName
+from credsweeper.ml_model.features.rule_severity import RuleSeverity
 from credsweeper.ml_model.features.search_in_attribute import SearchInAttribute
 from credsweeper.ml_model.features.word_in_path import WordInPath
 from credsweeper.ml_model.features.word_in_postamble import WordInPostamble

credsweeper/ml_model/features/morpheme_dense.py

@@ -7,13 +7,20 @@ class MorphemeDense(Feature):
     """Feature calculates morphemes density for a value"""
 
     def extract(self, candidate: Candidate) -> float:
+        density = 0.0
         if value := candidate.line_data_list[0].value.lower():
-            morphemes_counter = 0
+            morphemes_length = 0
             for morpheme in static_keyword_checklist.morpheme_set:
-                if morpheme in value:
-                    morphemes_counter += 1
+                morpheme_pos = value.find(morpheme)
+                if 0 <= morpheme_pos:
+                    morpheme_len = len(morpheme)
+                    while 0 <= morpheme_pos:
+                        morphemes_length += morpheme_len
+                        morpheme_pos += morpheme_len
+                        morpheme_pos = value.find(morpheme, morpheme_pos)
             # normalization: minimal morpheme length is 3
-            return 3.0 * morphemes_counter / len(value)
-        else:
-            # empty value case
-            return 0.0
+            density = morphemes_length / len(value)
+            if 1.0 < density:
+                # overlap morpheme case
+                density = 1.0
+        return density

credsweeper/ml_model/features/rule_severity.py

@@ -0,0 +1,21 @@
+from credsweeper.common.constants import Severity
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.ml_model.features.feature import Feature
+
+
+class RuleSeverity(Feature):
+    """Categorical feature that corresponds to rule name."""
+
+    def extract(self, candidate: Candidate) -> float:
+        if Severity.CRITICAL == candidate.severity:
+            return 1.0
+        elif Severity.HIGH == candidate.severity:
+            return 0.75
+        elif Severity.MEDIUM == candidate.severity:
+            return 0.5
+        elif Severity.LOW == candidate.severity:
+            return 0.25
+        elif Severity.INFO == candidate.severity:
+            return 0.0
+        else:
+            raise ValueError(f"Unknown type of severity: {candidate.severity}")

credsweeper/ml_model/features/word_in_path.py

@@ -1,3 +1,4 @@
+import os.path
 from pathlib import Path
 from typing import List, Any
 
@@ -16,7 +17,9 @@ class WordInPath(WordIn):
             path = Path(file_path)
             # apply ./ for normalised path to detect "/src" for relative path
             posix_lower_path = path.as_posix().lower() if path.is_absolute() else f"./{path.as_posix().lower()}"
-            return self.word_in_str(posix_lower_path)
+            # prevent extra confusion from the same word in extension
+            path_without_extension, _ = os.path.splitext(posix_lower_path)
+            return self.word_in_str(path_without_extension)
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])