credsweeper 1.11.5__py3-none-any.whl → 1.13.3__py3-none-any.whl

credsweeper/filters/value_pattern_check.py

@@ -1,10 +1,11 @@
 import re
+from typing import Optional
 
-from credsweeper.common.constants import DEFAULT_PATTERN_LEN
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.common.constants import DEFAULT_PATTERN_LEN, MAX_LINE_LENGTH
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValuePatternCheck(Filter):
@@ -22,36 +23,60 @@ class ValuePatternCheck(Filter):
     Default pattern LEN is 4
     """
 
-    def __init__(self, config: Config = None, pattern_len: int = DEFAULT_PATTERN_LEN):
+    MAX_PATTERN_LENGTH = int(MAX_LINE_LENGTH).bit_length()
+
+    def __init__(self, config: Optional[Config] = None, pattern_len: Optional[int] = None):
         """Create ValuePatternCheck with a specific pattern_len to check.
 
         Args:
             config: pattern len to use during check. DEFAULT_PATTERN_LEN by default
+            pattern_len: size of constant pattern length for any value size or None for dynamic pattern size
 
         """
-        self.pattern_len = pattern_len
-        # use non whitespace symbol pattern
-        self.pattern = re.compile(fr"(\S)\1{{{str(self.pattern_len - 1)},}}")
+        patterns_count = 1 + ValuePatternCheck.MAX_PATTERN_LENGTH
+        if pattern_len is None:
+            self.pattern_len = -1
+            # pattern length depends on value length
+            self.pattern_lengths = [max(x, DEFAULT_PATTERN_LEN) for x in range(patterns_count)]
+            self.patterns = [ValuePatternCheck.get_pattern(x) for x in range(patterns_count)]
+        elif isinstance(pattern_len, int) and DEFAULT_PATTERN_LEN <= pattern_len:
+            self.pattern_len = pattern_len
+            # constant pattern for any value length
+            self.pattern_lengths = [pattern_len] * patterns_count
+            self.patterns = [ValuePatternCheck.get_pattern(pattern_len)] * patterns_count
+        else:
+            raise ValueError(f"Wrong type of pattern length {type(pattern_len)} = {repr(pattern_len)}")
+
+    @staticmethod
+    def get_pattern(pattern_len: int) -> re.Pattern:
+        """Creates regex pattern to find N or more identical characters in sequence"""
+        if DEFAULT_PATTERN_LEN < pattern_len:
+            pattern = fr"(\S)\1{{{str(pattern_len - 1)},}}"
+        else:
+            pattern = r"(\S)\1{3,}"
+        return re.compile(pattern)
 
-    def equal_pattern_check(self, value: str) -> bool:
+    def equal_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value contain 4 and more same chars or numbers sequences.
 
         Args:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             True if contain and False if not
 
         """
-        if self.pattern.findall(value):
+        if self.patterns[bit_length].findall(value):
             return True
         return False
 
-    def ascending_pattern_check(self, value: str) -> bool:
+    def ascending_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value contain 4 and more ascending chars or numbers sequences.
 
         Arg:
             value: credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             True if contain and False if not
@@ -64,15 +89,16 @@ class ValuePatternCheck(Filter):
             else:
                 count = 1
                 continue
-            if count == self.pattern_len:
+            if count == self.pattern_lengths[bit_length]:
                 return True
         return False
 
-    def descending_pattern_check(self, value: str) -> bool:
+    def descending_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value contain 4 and more descending chars or numbers sequences.
 
         Arg:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             boolean variable. True if contain and False if not
@@ -85,59 +111,44 @@ class ValuePatternCheck(Filter):
             else:
                 count = 1
                 continue
-            if count == self.pattern_len:
+            if count == self.pattern_lengths[bit_length]:
                 return True
         return False
 
-    def check_val(self, value: str) -> bool:
+    def check_val(self, value: str, bit_length: int) -> bool:
         """Cumulative value check.
 
         Arg:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             boolean variable. True if contain and False if not
 
         """
-        if self.equal_pattern_check(value):
+        if self.equal_pattern_check(value, bit_length):
             return True
-        if self.ascending_pattern_check(value):
+        if self.ascending_pattern_check(value, bit_length):
             return True
-        if self.descending_pattern_check(value):
+        if self.descending_pattern_check(value, bit_length):
             return True
         return False
 
-    def duple_pattern_check(self, value: str) -> bool:
+    def duple_pattern_check(self, value: str, bit_length: int) -> bool:
         """Check if candidate value is a duplet value with possible patterns.
 
         Arg:
             value: string variable, credential candidate value
+            bit_length: speedup for len(value).bit_length()
 
         Return:
             boolean variable. True if contain and False if not
 
         """
-        # 001122334455... case
-        pair_duple = True
-        # 0102030405... case
-        even_duple = True
-        even_prev = value[0]
         even_value = value[0::2]
-        # 1020304050... case
-        odd_duple = True
-        odd_prev = value[1]
         odd_value = value[1::2]
-        for even_i, odd_i in zip(even_value, odd_value):
-            pair_duple &= even_i == odd_i
-            even_duple &= even_i == even_prev
-            odd_duple &= odd_i == odd_prev
-            if not pair_duple and not even_duple and not odd_duple:
-                break
-        else:
-            if pair_duple or odd_duple:
-                return self.check_val(even_value)
-            if even_duple:
-                return self.check_val(odd_value)
+        if self.check_val(even_value, bit_length) and self.check_val(odd_value, bit_length):
+            return True
         return False
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
@@ -151,13 +162,22 @@ class ValuePatternCheck(Filter):
             boolean variable. True, if need to filter candidate and False if left
 
         """
-        if len(line_data.value) < self.pattern_len:
+        value_length = len(line_data.value)
+        bit_length = max(DEFAULT_PATTERN_LEN, value_length.bit_length())
+
+        if ValuePatternCheck.MAX_PATTERN_LENGTH < bit_length:
+            # huge values may contain anything
+            return False
+
+        if 0 <= value_length < self.pattern_len or value_length < self.pattern_lengths[bit_length]:
+            # too short value
             return True
 
-        if self.check_val(line_data.value):
+        if self.check_val(line_data.value, bit_length):
             return True
 
-        if 2 * self.pattern_len <= len(line_data.value) and self.duple_pattern_check(line_data.value):
+        if 2 * self.pattern_lengths[bit_length] <= value_length \
+                and self.duple_pattern_check(line_data.value, bit_length):
             return True
 
         return False

credsweeper/filters/value_similarity_check.py

@@ -1,13 +1,15 @@
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from typing import Optional
+
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueSimilarityCheck(Filter):
     """Check if candidate value is at least 70% same as candidate keyword. Like: `secret = "mysecret"`."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_split_keyword_check.py

@@ -1,16 +1,17 @@
+from typing import Optional
 from typing import Union
 
 from credsweeper.common import static_keyword_checklist
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueSplitKeywordCheck(Filter):
     """Check value by splitting with standard whitespace separators and any word is not matched in checklist."""
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/filters/value_string_type_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueStringTypeCheck(Filter):
@@ -23,10 +24,11 @@ class ValueStringTypeCheck(Filter):
     False otherwise
     """
 
-    MULTIBYTE_PATTERN = re.compile(r"(\s*(0x)?[0-9a-f]{1,3}\s*,){8,80}", flags=re.IGNORECASE)
+    MULTIBYTE_PATTERN = re.compile(r"((0x)?[0-9a-f]{1,16}[UL]*)(\s*,\s*((0x)?[0-9a-f]{1,16}[UL]*)){3}",
+                                   flags=re.IGNORECASE)
 
-    def __init__(self, config: Config) -> None:
-        self.check_for_literals = config.check_for_literals
+    def __init__(self, config: Optional[Config] = None, check_for_literals=True) -> None:
+        self.check_for_literals = check_for_literals
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
         """Run filter checks on received credential candidate data 'line_data'.
@@ -42,13 +44,14 @@ class ValueStringTypeCheck(Filter):
         if not self.check_for_literals or line_data.url_part:
             return False
 
-        if ValueStringTypeCheck.MULTIBYTE_PATTERN.match(line_data.value):
+        if ValueStringTypeCheck.MULTIBYTE_PATTERN.search(line_data.value):
             return False
 
         if line_data.is_source_file_with_quotes() \
                 and not line_data.is_comment() \
                 and not line_data.is_well_quoted_value \
                 and not line_data.is_quoted \
+                and not '0' <= line_data.value[0] <= '9' \
                 and line_data.separator and '=' in line_data.separator:
             # heterogeneous code e.g. YAML in Python uses colon sign instead equals
             return True

credsweeper/filters/value_token_base_check.py

@@ -1,11 +1,12 @@
 import contextlib
 from abc import abstractmethod
+from typing import Optional
 from typing import Tuple
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 from credsweeper.utils.hop_stat import HopStat
 
 
@@ -26,7 +27,7 @@ class ValueTokenBaseCheck(Filter):
         64: 2.15981241,
     }
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         self.__hop_stat = HopStat()
 
     @staticmethod

credsweeper/filters/value_token_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueTokenCheck(Filter):
@@ -17,9 +18,9 @@ class ValueTokenCheck(Filter):
 
     """
 
-    SPLIT_PATTERN = r"(?<!,) (?!,)|;|\)|\(|{|}|<|>|\[|\]|`"
+    SPLIT_PATTERN = re.compile(r"(?<!\W) (?!\W)|[;(){}<>[\]`]")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/logger/__init__.py

@@ -1 +0,0 @@
-from credsweeper.logger.logger import Logger

credsweeper/logger/logger.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Optional
 
 from credsweeper.app import APP_PATH
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 
 class Logger:

credsweeper/ml_model/__init__.py

@@ -1 +0,0 @@
-from credsweeper.ml_model.ml_validator import MlValidator

credsweeper/ml_model/features/__init__.py

@@ -5,6 +5,7 @@ from credsweeper.ml_model.features.is_secret_numeric import IsSecretNumeric
 from credsweeper.ml_model.features.length_of_attribute import LengthOfAttribute
 from credsweeper.ml_model.features.morpheme_dense import MorphemeDense
 from credsweeper.ml_model.features.rule_name import RuleName
+from credsweeper.ml_model.features.rule_severity import RuleSeverity
 from credsweeper.ml_model.features.search_in_attribute import SearchInAttribute
 from credsweeper.ml_model.features.word_in_path import WordInPath
 from credsweeper.ml_model.features.word_in_postamble import WordInPostamble

credsweeper/ml_model/features/entropy_evaluation.py

@@ -4,7 +4,7 @@ from typing import Dict, List, Set
 import numpy as np
 
 from credsweeper.common.constants import Chars, ML_HUNK
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.file_handler.data_content_provider import MIN_DATA_LEN
 from credsweeper.ml_model.features.feature import Feature
 

credsweeper/ml_model/features/feature.py

@@ -3,14 +3,14 @@ from typing import List, Any
 
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 
 
 class Feature(ABC):
     """Base class for features."""
 
     def __init__(self):
-        self.words = []
+        pass
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         """Call base class for features.
@@ -25,20 +25,3 @@ class Feature(ABC):
     def extract(self, candidate: Candidate) -> Any:
         """Abstract method of base class"""
         raise NotImplementedError
-
-    @property
-    def words(self) -> List[str]:
-        """getter"""
-        return self.__words
-
-    @words.setter
-    def words(self, words: List[str]) -> None:
-        """setter"""
-        self.__words = words
-
-    def any_word_in_(self, a_string: str) -> bool:
-        """Returns true if any words in a string"""
-        for i in self.words:
-            if i in a_string:
-                return True
-        return False

credsweeper/ml_model/features/file_extension.py

@@ -2,7 +2,7 @@ from typing import List, Any
 
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
 
 
@@ -19,7 +19,7 @@ class FileExtension(WordIn):
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         extension_set = set(candidate.line_data_list[0].file_type.lower() for candidate in candidates)
-        return self.word_in_set(extension_set)
+        return self.word_in_(extension_set)
 
     def extract(self, candidate: Candidate) -> Any:
         raise NotImplementedError

credsweeper/ml_model/features/has_html_tag.py

@@ -1,17 +1,18 @@
 from credsweeper.common.constants import CHUNK_SIZE
-from credsweeper.credentials import Candidate
-from credsweeper.ml_model.features.feature import Feature
-from credsweeper.utils import Util
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.ml_model.features.word_in import WordIn
+from credsweeper.utils.util import Util
 
 
-class HasHtmlTag(Feature):
+class HasHtmlTag(WordIn):
     """Feature is true if line has HTML tags (HTML file)."""
 
+    HTML_WORDS = [
+        '< img', '<img', '< script', '<script', '< p', '<p', '< link', '<link', '< meta', '<meta', '< a', '<a'
+    ]
+
     def __init__(self) -> None:
-        super().__init__()
-        self.words = [
-            '< img', '<img', '< script', '<script', '< p', '<p', '< link', '<link', '< meta', '<meta', '< a', '<a'
-        ]
+        super().__init__(HasHtmlTag.HTML_WORDS)
 
     def extract(self, candidate: Candidate) -> bool:
         subtext = Util.subtext(candidate.line_data_list[0].line, candidate.line_data_list[0].value_start, CHUNK_SIZE)
@@ -19,8 +20,9 @@ class HasHtmlTag(Feature):
         if '<' not in candidate_line_data_list_0_line_lower:
             # early check
             return False
-        if self.any_word_in_(candidate_line_data_list_0_line_lower):
-            return True
+        for i in self.words:
+            if i in candidate_line_data_list_0_line_lower:
+                return True
         if "/>" in candidate_line_data_list_0_line_lower or "</" in candidate_line_data_list_0_line_lower:
             # possible closed tag
             return True

credsweeper/ml_model/features/is_secret_numeric.py

@@ -1,4 +1,6 @@
-from credsweeper.credentials import Candidate
+import contextlib
+
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 
@@ -6,8 +8,7 @@ class IsSecretNumeric(Feature):
     """Feature is true if candidate value is a numerical value."""
 
     def extract(self, candidate: Candidate) -> bool:
-        try:
+        with contextlib.suppress(ValueError):
             float(candidate.line_data_list[0].value)
             return True
-        except ValueError:
-            return False
+        return False

credsweeper/ml_model/features/length_of_attribute.py

@@ -1,7 +1,7 @@
 import numpy as np
 
 from credsweeper.common.constants import ML_HUNK
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 

credsweeper/ml_model/features/morpheme_dense.py

@@ -1,5 +1,5 @@
 from credsweeper.common import static_keyword_checklist
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 
@@ -7,13 +7,20 @@ class MorphemeDense(Feature):
     """Feature calculates morphemes density for a value"""
 
     def extract(self, candidate: Candidate) -> float:
+        density = 0.0
         if value := candidate.line_data_list[0].value.lower():
-            morphemes_counter = 0
+            morphemes_length = 0
             for morpheme in static_keyword_checklist.morpheme_set:
-                if morpheme in value:
-                    morphemes_counter += 1
+                morpheme_pos = value.find(morpheme)
+                if 0 <= morpheme_pos:
+                    morpheme_len = len(morpheme)
+                    while 0 <= morpheme_pos:
+                        morphemes_length += morpheme_len
+                        morpheme_pos += morpheme_len
+                        morpheme_pos = value.find(morpheme, morpheme_pos)
             # normalization: minimal morpheme length is 3
-            return 3.0 * morphemes_counter / len(value)
-        else:
-            # empty value case
-            return 0.0
+            density = morphemes_length / len(value)
+            if 1.0 < density:
+                # overlap morpheme case
+                density = 1.0
+        return density

credsweeper/ml_model/features/rule_name.py

@@ -2,7 +2,7 @@ from typing import List, Any
 
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
 
 
@@ -19,7 +19,7 @@ class RuleName(WordIn):
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         candidate_rule_set = set(x.rule_name for x in candidates)
-        return self.word_in_set(candidate_rule_set)
+        return self.word_in_(candidate_rule_set)
 
     def extract(self, candidate: Candidate) -> Any:
         raise NotImplementedError

credsweeper/ml_model/features/rule_severity.py

@@ -0,0 +1,21 @@
+from credsweeper.common.constants import Severity
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.ml_model.features.feature import Feature
+
+
+class RuleSeverity(Feature):
+    """Categorical feature that corresponds to rule name."""
+
+    def extract(self, candidate: Candidate) -> float:
+        if Severity.CRITICAL == candidate.severity:
+            return 1.0
+        elif Severity.HIGH == candidate.severity:
+            return 0.75
+        elif Severity.MEDIUM == candidate.severity:
+            return 0.5
+        elif Severity.LOW == candidate.severity:
+            return 0.25
+        elif Severity.INFO == candidate.severity:
+            return 0.0
+        else:
+            raise ValueError(f"Unknown type of severity: {candidate.severity}")

credsweeper/ml_model/features/search_in_attribute.py

@@ -1,6 +1,6 @@
 import re
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 

credsweeper/ml_model/features/word_in.py

@@ -1,9 +1,9 @@
 from abc import abstractmethod
-from typing import List, Any, Tuple, Set
+from typing import List, Any, Set, Union
 
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 
@@ -18,42 +18,19 @@ class WordIn(Feature):
         if len(self.enumerated_words) != self.dimension:
             raise RuntimeError(f"Check duplicates:{words}")
 
-    @property
-    def enumerated_words(self) -> List[Tuple[int, str]]:
-        """getter for speedup"""
-        return self.__enumerated_words
-
-    @enumerated_words.setter
-    def enumerated_words(self, enumerated_words: List[Tuple[int, str]]) -> None:
-        """setter for speedup"""
-        self.__enumerated_words = enumerated_words
-
-    @property
-    def dimension(self) -> int:
-        """getter"""
-        return self.__dimension
-
-    @dimension.setter
-    def dimension(self, dimension: int) -> None:
-        """setter"""
-        self.__dimension = dimension
-
     @abstractmethod
     def extract(self, candidate: Candidate) -> Any:
         raise NotImplementedError
 
-    def word_in_str(self, a_string: str) -> np.ndarray:
-        """Returns array with words included in a string"""
-        result: np.ndarray = np.zeros(shape=[self.dimension], dtype=np.int8)
-        for i, word in self.enumerated_words:
-            if word in a_string:
-                result[i] = 1
-        return np.array([result])
+    @property
+    def zero(self) -> np.ndarray:
+        """Returns zero filled array for case of empty input"""
+        return np.zeros(shape=[self.dimension], dtype=np.int8)
 
-    def word_in_set(self, a_strings_set: Set[str]) -> np.ndarray:
-        """Returns array with words matches in a_strings_set"""
-        result: np.ndarray = np.zeros(shape=[self.dimension], dtype=np.int8)
+    def word_in_(self, iterable_data: Union[str, List[str], Set[str]]) -> np.ndarray:
+        """Returns array with words included in a string"""
+        result: np.ndarray = self.zero
         for i, word in self.enumerated_words:
-            if word in a_strings_set:
+            if word in iterable_data:
                 result[i] = 1
         return np.array([result])