credsweeper 1.12.2__py3-none-any.whl → 1.13.1__py3-none-any.whl

credsweeper/filters/group/token_pattern.py

@@ -1,6 +1,6 @@
 from credsweeper.common.constants import GroupType
 from credsweeper.config.config import Config
-from credsweeper.filters import ValueCoupleKeywordCheck, ValueCamelCaseCheck, ValueNumberCheck, ValuePatternCheck
+from credsweeper.filters import ValueMorphemesCheck, ValueCamelCaseCheck, ValueNumberCheck, ValuePatternCheck
 from credsweeper.filters.group.group import Group
 
 
@@ -10,7 +10,7 @@ class TokenPattern(Group):
     def __init__(self, config: Config) -> None:
         super().__init__(config, GroupType.DEFAULT)
         self.filters = [
-            ValueCoupleKeywordCheck(),
+            ValueMorphemesCheck(),
             ValueNumberCheck(),
             ValueCamelCaseCheck(),
             ValuePatternCheck(),

credsweeper/filters/group/weird_base36_token.py

@@ -1,6 +1,6 @@
 from credsweeper.common.constants import GroupType
 from credsweeper.config.config import Config
-from credsweeper.filters import ValueCoupleKeywordCheck, ValuePatternCheck, ValueNumberCheck, ValueEntropyBase36Check, \
+from credsweeper.filters import ValueMorphemesCheck, ValuePatternCheck, ValueNumberCheck, ValueEntropyBase36Check, \
     ValueTokenBase36Check
 from credsweeper.filters.group.group import Group
 
@@ -11,7 +11,7 @@ class WeirdBase36Token(Group):
     def __init__(self, config: Config) -> None:
         super().__init__(config, GroupType.DEFAULT)
         self.filters = [
-            ValueCoupleKeywordCheck(),
+            ValueMorphemesCheck(threshold=1),
             ValuePatternCheck(),
             ValueNumberCheck(),
             ValueTokenBase36Check(),

credsweeper/filters/group/weird_base64_token.py

@@ -1,6 +1,6 @@
 from credsweeper.common.constants import GroupType
 from credsweeper.config.config import Config
-from credsweeper.filters import ValueCoupleKeywordCheck, ValueNotPartEncodedCheck, \
+from credsweeper.filters import ValueMorphemesCheck, ValueNotPartEncodedCheck, \
     ValueBase64DataCheck, ValueEntropyBase64Check, ValuePatternCheck, ValueNumberCheck, ValueTokenBase64Check, \
     ValueBase64PartCheck
 from credsweeper.filters.group.group import Group
@@ -12,7 +12,7 @@ class WeirdBase64Token(Group):
     def __init__(self, config: Config) -> None:
         super().__init__(config, GroupType.DEFAULT)
         self.filters = [
-            ValueCoupleKeywordCheck(),
+            ValueMorphemesCheck(threshold=1),
             ValueNumberCheck(),
             ValueBase64DataCheck(),
             ValueTokenBase64Check(),

credsweeper/filters/value_file_path_check.py

@@ -35,6 +35,8 @@ class ValueFilePathCheck(Filter):
 
         """
         value = line_data.value
+        bit_length = len(value).bit_length()
+        morpheme_threshold = 1 if 6 > bit_length else bit_length - 4
         contains_unix_separator = '/' in value
         if contains_unix_separator:
             if ("://" in value  #
@@ -45,14 +47,14 @@ class ValueFilePathCheck(Filter):
                     or value.startswith("//") and ':' == line_data.separator):
                 # common case for url definition or aliases
                 # or _keyword_://example.com where : is the separator
-                return static_keyword_checklist.check_morphemes(value.lower(), 1)
+                return static_keyword_checklist.check_morphemes(value.lower(), morpheme_threshold)
             # base64 encoded data might look like linux path
             min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(value))
             # get minimal entropy to compare with shannon entropy of found value
             # min_entropy == 0 means that the value cannot be checked with the entropy due high variance
             for i in value:
                 if i not in self.base64stdpad_possible_set:
-                    # value contains wrong BASE64STDPAD_CHARS symbols like -_
+                    # value contains wrong BASE64STDPAD_CHARS symbols like -_.
                     break
             else:
                 # all symbols are from base64 alphabet
@@ -74,5 +76,5 @@ class ValueFilePathCheck(Filter):
                     break
             else:
                 if contains_unix_separator ^ contains_windows_separator:
-                    return static_keyword_checklist.check_morphemes(value.lower(), 1)
+                    return static_keyword_checklist.check_morphemes(value.lower(), morpheme_threshold)
         return False

credsweeper/filters/value_github_check.py

@@ -12,7 +12,7 @@ from credsweeper.filters.filter import Filter
 
 
 class ValueGitHubCheck(Filter):
-    """GitHub Classic Token validation"""
+    """NPM or GitHub Classic Token validation"""
 
     def __init__(self, config: Optional[Config] = None) -> None:
         pass
@@ -29,8 +29,9 @@ class ValueGitHubCheck(Filter):
 
         """
         # https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
+        # https://github.blog/security/announcing-npms-new-access-token-format/
         with contextlib.suppress(Exception):
-            if line_data.value.startswith("gh") and '_' == line_data.value[3]:
+            if (line_data.value.startswith("gh") and '_' == line_data.value[3]) or line_data.value.startswith("npm_"):
                 token = line_data.value[4:-6]
                 data = token.encode(ASCII, errors="strict")
                 crc32sum = binascii.crc32(data)

credsweeper/filters/value_morphemes_check.py

@@ -0,0 +1,43 @@
+from typing import Optional
+
+from credsweeper.common import static_keyword_checklist
+from credsweeper.common.constants import MAX_LINE_LENGTH
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
+from credsweeper.file_handler.analysis_target import AnalysisTarget
+from credsweeper.filters.filter import Filter
+
+
+class ValueMorphemesCheck(Filter):
+    """Check value for a threshold of morphemes count"""
+
+    THRESHOLDS_X3 = int(MAX_LINE_LENGTH).bit_length()
+    # one morpheme is very likely to be random generated even for 3 symbols
+    MAX_MORPHEMES_LIMIT = max(1, THRESHOLDS_X3 - 4)
+
+    def __init__(self, config: Optional[Config] = None, threshold: Optional[int] = None) -> None:
+        # threshold - minimum morphemes number in a value
+        if threshold is None:
+            # use dynamic thresholds
+            self.thresholds = [max(1, x - 4) for x in range(ValueMorphemesCheck.THRESHOLDS_X3)]
+        elif isinstance(threshold, int) and 0 <= threshold:
+            # constant thresholds for any pattern
+            self.thresholds = [threshold] * ValueMorphemesCheck.THRESHOLDS_X3
+        else:
+            raise ValueError(f"Wrong type of pattern length {type(threshold)} = {repr(threshold)}")
+
+    def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
+        """Run filter checks on received credential candidate data 'line_data'.
+
+        Args:
+            line_data: credential candidate data
+            target: multiline target from which line data was obtained
+
+        Return:
+            True, if need to filter candidate and False if left
+
+        """
+        threshold_id = len(line_data.value).bit_length()
+        # use the last (max) threshold in very huge value
+        threshold = self.thresholds[threshold_id] if len(self.thresholds) > threshold_id else self.thresholds[-1]
+        return static_keyword_checklist.check_morphemes(line_data.value.lower(), threshold)

credsweeper/filters/value_string_type_check.py

@@ -51,6 +51,7 @@ class ValueStringTypeCheck(Filter):
                 and not line_data.is_comment() \
                 and not line_data.is_well_quoted_value \
                 and not line_data.is_quoted \
+                and not '0' <= line_data.value[0] <= '9' \
                 and line_data.separator and '=' in line_data.separator:
             # heterogeneous code e.g. YAML in Python uses colon sign instead equals
             return True

credsweeper/ml_model/features/feature.py

@@ -10,7 +10,7 @@ class Feature(ABC):
     """Base class for features."""
 
     def __init__(self):
-        self.words = []
+        pass
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         """Call base class for features.
@@ -25,20 +25,3 @@ class Feature(ABC):
     def extract(self, candidate: Candidate) -> Any:
         """Abstract method of base class"""
         raise NotImplementedError
-
-    @property
-    def words(self) -> List[str]:
-        """getter"""
-        return self.__words
-
-    @words.setter
-    def words(self, words: List[str]) -> None:
-        """setter"""
-        self.__words = words
-
-    def any_word_in_(self, a_string: str) -> bool:
-        """Returns true if any words in a string"""
-        for i in self.words:
-            if i in a_string:
-                return True
-        return False

credsweeper/ml_model/features/file_extension.py

@@ -19,7 +19,7 @@ class FileExtension(WordIn):
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         extension_set = set(candidate.line_data_list[0].file_type.lower() for candidate in candidates)
-        return self.word_in_set(extension_set)
+        return self.word_in_(extension_set)
 
     def extract(self, candidate: Candidate) -> Any:
         raise NotImplementedError

credsweeper/ml_model/features/has_html_tag.py

@@ -1,17 +1,18 @@
 from credsweeper.common.constants import CHUNK_SIZE
 from credsweeper.credentials.candidate import Candidate
-from credsweeper.ml_model.features.feature import Feature
+from credsweeper.ml_model.features.word_in import WordIn
 from credsweeper.utils.util import Util
 
 
-class HasHtmlTag(Feature):
+class HasHtmlTag(WordIn):
     """Feature is true if line has HTML tags (HTML file)."""
 
+    HTML_WORDS = [
+        '< img', '<img', '< script', '<script', '< p', '<p', '< link', '<link', '< meta', '<meta', '< a', '<a'
+    ]
+
     def __init__(self) -> None:
-        super().__init__()
-        self.words = [
-            '< img', '<img', '< script', '<script', '< p', '<p', '< link', '<link', '< meta', '<meta', '< a', '<a'
-        ]
+        super().__init__(HasHtmlTag.HTML_WORDS)
 
     def extract(self, candidate: Candidate) -> bool:
         subtext = Util.subtext(candidate.line_data_list[0].line, candidate.line_data_list[0].value_start, CHUNK_SIZE)
@@ -19,8 +20,9 @@ class HasHtmlTag(Feature):
         if '<' not in candidate_line_data_list_0_line_lower:
             # early check
             return False
-        if self.any_word_in_(candidate_line_data_list_0_line_lower):
-            return True
+        for i in self.words:
+            if i in candidate_line_data_list_0_line_lower:
+                return True
         if "/>" in candidate_line_data_list_0_line_lower or "</" in candidate_line_data_list_0_line_lower:
             # possible closed tag
             return True

credsweeper/ml_model/features/is_secret_numeric.py

@@ -1,3 +1,5 @@
+import contextlib
+
 from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
@@ -6,8 +8,7 @@ class IsSecretNumeric(Feature):
     """Feature is true if candidate value is a numerical value."""
 
     def extract(self, candidate: Candidate) -> bool:
-        try:
+        with contextlib.suppress(ValueError):
             float(candidate.line_data_list[0].value)
             return True
-        except ValueError:
-            return False
+        return False

credsweeper/ml_model/features/rule_name.py

@@ -19,7 +19,7 @@ class RuleName(WordIn):
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         candidate_rule_set = set(x.rule_name for x in candidates)
-        return self.word_in_set(candidate_rule_set)
+        return self.word_in_(candidate_rule_set)
 
     def extract(self, candidate: Candidate) -> Any:
         raise NotImplementedError

credsweeper/ml_model/features/word_in.py

@@ -1,5 +1,5 @@
 from abc import abstractmethod
-from typing import List, Any, Tuple, Set
+from typing import List, Any, Set, Union
 
 import numpy as np
 
@@ -18,42 +18,19 @@ class WordIn(Feature):
         if len(self.enumerated_words) != self.dimension:
             raise RuntimeError(f"Check duplicates:{words}")
 
-    @property
-    def enumerated_words(self) -> List[Tuple[int, str]]:
-        """getter for speedup"""
-        return self.__enumerated_words
-
-    @enumerated_words.setter
-    def enumerated_words(self, enumerated_words: List[Tuple[int, str]]) -> None:
-        """setter for speedup"""
-        self.__enumerated_words = enumerated_words
-
-    @property
-    def dimension(self) -> int:
-        """getter"""
-        return self.__dimension
-
-    @dimension.setter
-    def dimension(self, dimension: int) -> None:
-        """setter"""
-        self.__dimension = dimension
-
     @abstractmethod
     def extract(self, candidate: Candidate) -> Any:
         raise NotImplementedError
 
-    def word_in_str(self, a_string: str) -> np.ndarray:
-        """Returns array with words included in a string"""
-        result: np.ndarray = np.zeros(shape=[self.dimension], dtype=np.int8)
-        for i, word in self.enumerated_words:
-            if word in a_string:
-                result[i] = 1
-        return np.array([result])
+    @property
+    def zero(self) -> np.ndarray:
+        """Returns zero filled array for case of empty input"""
+        return np.zeros(shape=[self.dimension], dtype=np.int8)
 
-    def word_in_set(self, a_strings_set: Set[str]) -> np.ndarray:
-        """Returns array with words matches in a_strings_set"""
-        result: np.ndarray = np.zeros(shape=[self.dimension], dtype=np.int8)
+    def word_in_(self, iterable_data: Union[str, List[str], Set[str]]) -> np.ndarray:
+        """Returns array with words included in a string"""
+        result: np.ndarray = self.zero
         for i, word in self.enumerated_words:
-            if word in a_strings_set:
+            if word in iterable_data:
                 result[i] = 1
         return np.array([result])

credsweeper/ml_model/features/word_in_path.py

@@ -19,9 +19,8 @@ class WordInPath(WordIn):
             posix_lower_path = path.as_posix().lower() if path.is_absolute() else f"./{path.as_posix().lower()}"
             # prevent extra confusion from the same word in extension
             path_without_extension, _ = os.path.splitext(posix_lower_path)
-            return self.word_in_str(path_without_extension)
-        else:
-            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])
+            return self.word_in_(path_without_extension)
+        return np.array([self.zero])
 
     def extract(self, candidate: Candidate) -> Any:
         raise NotImplementedError

credsweeper/ml_model/features/word_in_postamble.py

@@ -15,7 +15,4 @@ class WordInPostamble(WordIn):
             else candidate.line_data_list[0].value_end + ML_HUNK
         postamble = candidate.line_data_list[0].line[candidate.line_data_list[0].value_end:postamble_end].strip()
 
-        if postamble:
-            return self.word_in_str(postamble.lower())
-        else:
-            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])
+        return self.word_in_(postamble.lower()) if postamble else np.array([self.zero])

credsweeper/ml_model/features/word_in_preamble.py

@@ -20,7 +20,4 @@ class WordInPreamble(WordIn):
                 else candidate.line_data_list[0].value_start - ML_HUNK
             preamble = candidate.line_data_list[0].line[preamble_start:candidate.line_data_list[0].value_start].strip()
 
-        if preamble:
-            return self.word_in_str(preamble.lower())
-        else:
-            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])
+        return self.word_in_(preamble.lower()) if preamble else np.array([self.zero])

credsweeper/ml_model/features/word_in_transition.py

@@ -15,7 +15,4 @@ class WordInTransition(WordIn):
         else:
             transition = ''
 
-        if transition:
-            return self.word_in_str(transition.lower())
-        else:
-            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])
+        return self.word_in_(transition.lower()) if transition else np.array([self.zero])

credsweeper/ml_model/features/word_in_value.py

@@ -10,6 +10,5 @@ class WordInValue(WordIn):
     def extract(self, candidate: Candidate) -> np.ndarray:
         """Returns array of matching words for first line"""
         if value := candidate.line_data_list[0].value:
-            return self.word_in_str(value.lower())
-        else:
-            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])
+            return self.word_in_(value.lower())
+        return np.array([self.zero])

credsweeper/ml_model/features/word_in_variable.py

@@ -10,6 +10,5 @@ class WordInVariable(WordIn):
     def extract(self, candidate: Candidate) -> np.ndarray:
         """Returns array of matching words for first line"""
         if variable := candidate.line_data_list[0].variable:
-            return self.word_in_str(variable.lower())
-        else:
-            return np.zeros(shape=[self.dimension], dtype=np.int8)
+            return self.word_in_(variable.lower())
+        return np.array([self.zero])

credsweeper/ml_model/ml_config.json

@@ -10,6 +10,7 @@
     "features": [
         {
             "type": "RuleSeverity",
+            "comment": "INFO=0.0, LOW=0.25, MEDIUM=0.5, HIGH=0.75, CRITICAL=1.0",
             "kwargs": {}
         },
         {
@@ -62,7 +63,7 @@
             "type": "SearchInAttribute",
             "comment": "Repeated symbol",
             "kwargs": {
-                "pattern": ".*(?:(\\S)(\\S))((\\1.)|(.\\2)){7,}",
+                "pattern": "(?:(\\S)(\\S))((\\1.)|(.\\2)){7,}",
                 "attribute": "value"
             }
         },
@@ -70,7 +71,7 @@
             "type": "SearchInAttribute",
             "comment": "SHA marker",
             "kwargs": {
-                "pattern": ".*(?i:sha)[_-]?(224|256|384|512)",
+                "pattern": "(?i:sha)[_-]?(224|256|384|512)",
                 "attribute": "value"
             }
         },
@@ -126,7 +127,7 @@
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedNameCheck",
             "kwargs": {
-                "pattern": "(?i:(filters?|pub(lic)?)_?key)",
+                "pattern": "(?i:(sha[_-]?(224|256|384|512)|projects?|filters?|pub(lic)?)_?key)",
                 "attribute": "variable"
             }
         },
@@ -134,7 +135,7 @@
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedNameCheck",
             "kwargs": {
-                "pattern": "(?i:(id|size|name|type|manager|algorithm|view|error)$)",
+                "pattern": "(?i:(id|sum|size|name|type|manager|algorithm|pattern|view|error|date(time)?|time(stamp)?|tag|version|hash|rate)$)",
                 "attribute": "variable"
             }
         },
@@ -245,8 +246,10 @@
                     "crypt",
                     "crypted",
                     "decrypt",
+                    "edited",
                     "encrypt",
                     "example",
+                    "expire",
                     "fake",
                     "file",
                     "foo",
@@ -260,7 +263,8 @@
                     "pass",
                     "public",
                     "pwd",
-                    "rsa-",
+                    "redacted",
+                    "rsa",
                     "salt",
                     "secret",
                     "sha",
@@ -339,6 +343,7 @@
                     "get",
                     "e.g.",
                     "equal",
+                    "env",
                     "example",
                     "expect",
                     "line",
@@ -484,6 +489,7 @@
                     ".bat",
                     ".bats",
                     ".bazel",
+                    ".bin",
                     ".build",
                     ".bundle",
                     ".bzl",
@@ -504,7 +510,6 @@
                     ".csp",
                     ".csv",
                     ".dist",
-                    ".doc",
                     ".dockerfile",
                     ".edited",
                     ".eex",
@@ -527,6 +532,8 @@
                     ".gtpl",
                     ".h",
                     ".haml",
+                    ".har",
+                    ".hpp",
                     ".hs",
                     ".html",
                     ".idl",
@@ -657,8 +664,8 @@
                     "CMD Password",
                     "CMD Secret",
                     "CMD Token",
+                    "CURL User Password",
                     "Credential",
-                    "Github Old Token",
                     "Key",
                     "Nonce",
                     "Password",
@@ -671,4 +678,4 @@
             }
         }
     ]
-}
+}

credsweeper/ml_model/ml_validator.py

@@ -272,7 +272,7 @@ class MlValidator:
         if head != tail:
             probability[head:tail] = self._batch_call_model(line_input_list, variable_input_list, value_input_list,
                                                             features_list)
-        is_cred = probability > self.threshold
+        is_cred = self.threshold <= probability
         if logger.isEnabledFor(logging.DEBUG):
             for i, decision in enumerate(is_cred):
                 logger.debug("ML decision: %s with prediction: %s for value: %s", decision, probability[i],