credsweeper 1.11.5__py3-none-any.whl → 1.12.0__py3-none-any.whl

credsweeper/filters/value_string_type_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueStringTypeCheck(Filter):
@@ -23,10 +24,11 @@ class ValueStringTypeCheck(Filter):
     False otherwise
     """
 
-    MULTIBYTE_PATTERN = re.compile(r"(\s*(0x)?[0-9a-f]{1,3}\s*,){8,80}", flags=re.IGNORECASE)
+    MULTIBYTE_PATTERN = re.compile(r"((0x)?[0-9a-f]{1,16}[UL]*)(\s*,\s*((0x)?[0-9a-f]{1,16}[UL]*)){3}",
+                                   flags=re.IGNORECASE)
 
-    def __init__(self, config: Config) -> None:
-        self.check_for_literals = config.check_for_literals
+    def __init__(self, config: Optional[Config] = None, check_for_literals=True) -> None:
+        self.check_for_literals = check_for_literals
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
         """Run filter checks on received credential candidate data 'line_data'.
@@ -42,7 +44,7 @@ class ValueStringTypeCheck(Filter):
         if not self.check_for_literals or line_data.url_part:
             return False
 
-        if ValueStringTypeCheck.MULTIBYTE_PATTERN.match(line_data.value):
+        if ValueStringTypeCheck.MULTIBYTE_PATTERN.search(line_data.value):
             return False
 
         if line_data.is_source_file_with_quotes() \

credsweeper/filters/value_token_base_check.py

@@ -1,11 +1,12 @@
 import contextlib
 from abc import abstractmethod
+from typing import Optional
 from typing import Tuple
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 from credsweeper.utils.hop_stat import HopStat
 
 
@@ -26,7 +27,7 @@ class ValueTokenBaseCheck(Filter):
         64: 2.15981241,
     }
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         self.__hop_stat = HopStat()
 
     @staticmethod

credsweeper/filters/value_token_check.py

@@ -1,9 +1,10 @@
 import re
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
+from credsweeper.filters.filter import Filter
 
 
 class ValueTokenCheck(Filter):
@@ -17,9 +18,9 @@ class ValueTokenCheck(Filter):
 
     """
 
-    SPLIT_PATTERN = r"(?<!,) (?!,)|;|\)|\(|{|}|<|>|\[|\]|`"
+    SPLIT_PATTERN = re.compile(r"(?<!\W) (?!\W)|[;(){}<>[\]`]")
 
-    def __init__(self, config: Config = None) -> None:
+    def __init__(self, config: Optional[Config] = None) -> None:
         pass
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

credsweeper/logger/__init__.py

@@ -1 +0,0 @@
-from credsweeper.logger.logger import Logger

credsweeper/logger/logger.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Optional
 
 from credsweeper.app import APP_PATH
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 
 class Logger:

credsweeper/ml_model/__init__.py

@@ -1 +0,0 @@
-from credsweeper.ml_model.ml_validator import MlValidator

credsweeper/ml_model/features/__init__.py

@@ -5,6 +5,7 @@ from credsweeper.ml_model.features.is_secret_numeric import IsSecretNumeric
 from credsweeper.ml_model.features.length_of_attribute import LengthOfAttribute
 from credsweeper.ml_model.features.morpheme_dense import MorphemeDense
 from credsweeper.ml_model.features.rule_name import RuleName
+from credsweeper.ml_model.features.rule_severity import RuleSeverity
 from credsweeper.ml_model.features.search_in_attribute import SearchInAttribute
 from credsweeper.ml_model.features.word_in_path import WordInPath
 from credsweeper.ml_model.features.word_in_postamble import WordInPostamble

credsweeper/ml_model/features/entropy_evaluation.py

@@ -4,7 +4,7 @@ from typing import Dict, List, Set
 import numpy as np
 
 from credsweeper.common.constants import Chars, ML_HUNK
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.file_handler.data_content_provider import MIN_DATA_LEN
 from credsweeper.ml_model.features.feature import Feature
 

credsweeper/ml_model/features/feature.py

@@ -3,7 +3,7 @@ from typing import List, Any
 
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 
 
 class Feature(ABC):

credsweeper/ml_model/features/file_extension.py

@@ -2,7 +2,7 @@ from typing import List, Any
 
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
 
 

credsweeper/ml_model/features/has_html_tag.py

@@ -1,7 +1,7 @@
 from credsweeper.common.constants import CHUNK_SIZE
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 
 class HasHtmlTag(Feature):

credsweeper/ml_model/features/is_secret_numeric.py

@@ -1,4 +1,4 @@
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 

credsweeper/ml_model/features/length_of_attribute.py

@@ -1,7 +1,7 @@
 import numpy as np
 
 from credsweeper.common.constants import ML_HUNK
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 

credsweeper/ml_model/features/morpheme_dense.py

@@ -1,5 +1,5 @@
 from credsweeper.common import static_keyword_checklist
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 
@@ -7,13 +7,20 @@ class MorphemeDense(Feature):
     """Feature calculates morphemes density for a value"""
 
     def extract(self, candidate: Candidate) -> float:
+        density = 0.0
         if value := candidate.line_data_list[0].value.lower():
-            morphemes_counter = 0
+            morphemes_length = 0
             for morpheme in static_keyword_checklist.morpheme_set:
-                if morpheme in value:
-                    morphemes_counter += 1
+                morpheme_pos = value.find(morpheme)
+                if 0 <= morpheme_pos:
+                    morpheme_len = len(morpheme)
+                    while 0 <= morpheme_pos:
+                        morphemes_length += morpheme_len
+                        morpheme_pos += morpheme_len
+                        morpheme_pos = value.find(morpheme, morpheme_pos)
             # normalization: minimal morpheme length is 3
-            return 3.0 * morphemes_counter / len(value)
-        else:
-            # empty value case
-            return 0.0
+            density = morphemes_length / len(value)
+            if 1.0 < density:
+                # overlap morpheme case
+                density = 1.0
+        return density

credsweeper/ml_model/features/rule_name.py

@@ -2,7 +2,7 @@ from typing import List, Any
 
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
 
 

credsweeper/ml_model/features/rule_severity.py

@@ -0,0 +1,21 @@
+from credsweeper.common.constants import Severity
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.ml_model.features.feature import Feature
+
+
+class RuleSeverity(Feature):
+    """Categorical feature that corresponds to rule name."""
+
+    def extract(self, candidate: Candidate) -> float:
+        if Severity.CRITICAL == candidate.severity:
+            return 1.0
+        elif Severity.HIGH == candidate.severity:
+            return 0.75
+        elif Severity.MEDIUM == candidate.severity:
+            return 0.5
+        elif Severity.LOW == candidate.severity:
+            return 0.25
+        elif Severity.INFO == candidate.severity:
+            return 0.0
+        else:
+            raise ValueError(f"Unknown type of severity: {candidate.severity}")

credsweeper/ml_model/features/search_in_attribute.py

@@ -1,6 +1,6 @@
 import re
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 

credsweeper/ml_model/features/word_in.py

@@ -3,7 +3,7 @@ from typing import List, Any, Tuple, Set
 
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.feature import Feature
 
 

credsweeper/ml_model/features/word_in_path.py

@@ -1,9 +1,10 @@
+import os.path
 from pathlib import Path
 from typing import List, Any
 
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
 
 
@@ -16,7 +17,9 @@ class WordInPath(WordIn):
             path = Path(file_path)
             # apply ./ for normalised path to detect "/src" for relative path
             posix_lower_path = path.as_posix().lower() if path.is_absolute() else f"./{path.as_posix().lower()}"
-            return self.word_in_str(posix_lower_path)
+            # prevent extra confusion from the same word in extension
+            path_without_extension, _ = os.path.splitext(posix_lower_path)
+            return self.word_in_str(path_without_extension)
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])
 

credsweeper/ml_model/features/word_in_postamble.py

@@ -1,7 +1,7 @@
 import numpy as np
 
 from credsweeper.common.constants import ML_HUNK
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
 
 

credsweeper/ml_model/features/word_in_preamble.py

@@ -1,7 +1,7 @@
 import numpy as np
 
 from credsweeper.common.constants import ML_HUNK
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
 
 

credsweeper/ml_model/features/word_in_transition.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
 
 

credsweeper/ml_model/features/word_in_value.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
 
 

credsweeper/ml_model/features/word_in_variable.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.ml_model.features.word_in import WordIn