credsweeper 1.11.2__py3-none-any.whl → 1.11.4__py3-none-any.whl

credsweeper/filters/value_token_check.py

@@ -17,7 +17,7 @@ class ValueTokenCheck(Filter):
 
     """
 
-    SPLIT_PATTERN = r" |;|\)|\(|{|}|<|>|\[|\]|`"
+    SPLIT_PATTERN = r"(?<!,) (?!,)|;|\)|\(|{|}|<|>|\[|\]|`"
 
     def __init__(self, config: Config = None) -> None:
         pass

credsweeper/ml_model/features/file_extension.py

@@ -15,10 +15,10 @@ class FileExtension(WordIn):
     """
 
     def __init__(self, extensions: List[str]) -> None:
-        super().__init__(extensions)
+        super().__init__(words=extensions)
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
-        extension_set = set([candidate.line_data_list[0].file_type.lower() for candidate in candidates])
+        extension_set = set(candidate.line_data_list[0].file_type.lower() for candidate in candidates)
         return self.word_in_set(extension_set)
 
     def extract(self, candidate: Candidate) -> Any:

credsweeper/ml_model/features/morpheme_dense.py

@@ -6,10 +6,6 @@ from credsweeper.ml_model.features.feature import Feature
 class MorphemeDense(Feature):
     """Feature calculates morphemes density for a value"""
 
-    def __init__(self) -> None:
-        """Class initializer"""
-        super().__init__()
-
     def extract(self, candidate: Candidate) -> float:
         if value := candidate.line_data_list[0].value.lower():
             morphemes_counter = 0

credsweeper/ml_model/features/rule_name.py

@@ -15,7 +15,7 @@ class RuleName(WordIn):
     """
 
     def __init__(self, rule_names: List[str]) -> None:
-        super().__init__(rule_names)
+        super().__init__(words=rule_names)
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         candidate_rule_set = set(x.rule_name for x in candidates)

credsweeper/ml_model/features/word_in_path.py

@@ -10,15 +10,6 @@ from credsweeper.ml_model.features.word_in import WordIn
 class WordInPath(WordIn):
     """Categorical feature that corresponds to words in path (POSIX, lowercase)"""
 
-    def __init__(self, words: List[str]) -> None:
-        """WordInPath constructor
-
-        Args:
-            words: list of predefined words - MUST BE IN LOWER CASE & POSIX
-
-        """
-        super().__init__(words)
-
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         # actually there must be one path because the candidates are grouped before
         if file_path := candidates[0].line_data_list[0].path:

credsweeper/ml_model/features/word_in_postamble.py

@@ -1,5 +1,3 @@
-from typing import List
-
 import numpy as np
 
 from credsweeper.common.constants import ML_HUNK
@@ -10,15 +8,6 @@ from credsweeper.ml_model.features.word_in import WordIn
 class WordInPostamble(WordIn):
     """Feature is true if line contains at least one word from predefined list."""
 
-    def __init__(self, words: List[str]) -> None:
-        """Feature returns array of matching words
-
-        Args:
-            words: list of predefined words - MUST BE IN LOWER CASE
-
-        """
-        super().__init__(words)
-
     def extract(self, candidate: Candidate) -> np.ndarray:
         """Returns true if any words in a part of line after value"""
         postamble_end = len(candidate.line_data_list[0].line) \

credsweeper/ml_model/features/word_in_preamble.py

@@ -1,5 +1,3 @@
-from typing import List
-
 import numpy as np
 
 from credsweeper.common.constants import ML_HUNK
@@ -10,15 +8,6 @@ from credsweeper.ml_model.features.word_in import WordIn
 class WordInPreamble(WordIn):
     """Feature is true if line contains at least one word from predefined list."""
 
-    def __init__(self, words: List[str]) -> None:
-        """Feature returns array of matching words
-
-        Args:
-            words: list of predefined words - MUST BE IN LOWER CASE
-
-        """
-        super().__init__(words)
-
     def extract(self, candidate: Candidate) -> np.ndarray:
         """Returns true if any words in line before variable or value"""
         if 0 <= candidate.line_data_list[0].variable_start:

credsweeper/ml_model/features/word_in_transition.py

@@ -1,5 +1,3 @@
-from typing import List
-
 import numpy as np
 
 from credsweeper.credentials import Candidate
@@ -9,15 +7,6 @@ from credsweeper.ml_model.features.word_in import WordIn
 class WordInTransition(WordIn):
     """Feature is true if line contains at least one word from predefined list."""
 
-    def __init__(self, words: List[str]) -> None:
-        """Feature returns array of matching words
-
-        Args:
-            words: list of predefined words - MUST BE IN LOWER CASE
-
-        """
-        super().__init__(words)
-
     def extract(self, candidate: Candidate) -> np.ndarray:
         """Returns true if any words between variable and value"""
         if 0 <= candidate.line_data_list[0].variable_end < candidate.line_data_list[0].value_start:

credsweeper/ml_model/features/word_in_value.py

@@ -1,5 +1,3 @@
-from typing import List
-
 import numpy as np
 
 from credsweeper.credentials import Candidate
@@ -9,15 +7,6 @@ from credsweeper.ml_model.features.word_in import WordIn
 class WordInValue(WordIn):
     """Feature returns true if candidate value contains at least one word from predefined list."""
 
-    def __init__(self, words: List[str]) -> None:
-        """Feature is true if candidate value contains at least one predefined word.
-
-        Args:
-            words: list of predefined words - MUST BE IN LOWER CASE and SORTED (preferred)
-
-        """
-        super().__init__(words)
-
     def extract(self, candidate: Candidate) -> np.ndarray:
         """Returns array of matching words for first line"""
         if value := candidate.line_data_list[0].value:

credsweeper/ml_model/features/word_in_variable.py

@@ -1,5 +1,3 @@
-from typing import List
-
 import numpy as np
 
 from credsweeper.credentials import Candidate
@@ -9,15 +7,6 @@ from credsweeper.ml_model.features.word_in import WordIn
 class WordInVariable(WordIn):
     """Feature returns array of words matching in variable"""
 
-    def __init__(self, words: List[str]) -> None:
-        """Feature is true if candidate value contains at least one predefined word.
-
-        Args:
-            words: list of predefined words - MUST BE IN LOWER CASE
-
-        """
-        super().__init__(words)
-
     def extract(self, candidate: Candidate) -> np.ndarray:
         """Returns array of matching words for first line"""
         if variable := candidate.line_data_list[0].variable:

credsweeper/ml_model/ml_validator.py

@@ -1,10 +1,11 @@
 import hashlib
+import json
 import logging
 from pathlib import Path
 from typing import List, Tuple, Union, Optional, Dict
 
 import numpy as np
-import onnxruntime as ort
+from onnxruntime import InferenceSession
 
 import credsweeper.ml_model.features as features
 from credsweeper.common.constants import ThresholdPreset, ML_HUNK
@@ -22,6 +23,8 @@ class MlValidator:
     # applied for unknown characters
     FAKE_CHAR = '\x01'
 
+    _dir_path = Path(__file__).parent
+
     def __init__(
             self,  #
             threshold: Union[float, ThresholdPreset],  #
@@ -36,35 +39,36 @@ class MlValidator:
             ml_model: path to ml model
             ml_providers: coma separated list of providers https://onnxruntime.ai/docs/execution-providers/
         """
-        dir_path = Path(__file__).parent
+        self.__session: Optional[InferenceSession] = None
 
         if ml_config:
             ml_config_path = Path(ml_config)
         else:
-            ml_config_path = dir_path / "ml_config.json"
+            ml_config_path = MlValidator._dir_path / "ml_config.json"
         with open(ml_config_path, "rb") as f:
-            md5_config = hashlib.md5(f.read()).hexdigest()
+            __ml_config_data = f.read()
+
+        model_config = json.loads(__ml_config_data)
 
         if ml_model:
             ml_model_path = Path(ml_model)
         else:
-            ml_model_path = dir_path / "ml_model.onnx"
+            ml_model_path = MlValidator._dir_path / "ml_model.onnx"
         with open(ml_model_path, "rb") as f:
-            md5_model = hashlib.md5(f.read()).hexdigest()
+            self.__ml_model_data = f.read()
 
         if ml_providers:
-            providers = ml_providers.split(',')
+            self.providers = ml_providers.split(',')
         else:
-            providers = ["CPUExecutionProvider"]
-        self.model_session = ort.InferenceSession(ml_model_path, providers=providers)
+            self.providers = ["CPUExecutionProvider"]
 
-        model_config = Util.json_load(ml_config_path)
         if isinstance(threshold, float):
             self.threshold = threshold
         elif isinstance(threshold, ThresholdPreset) and "thresholds" in model_config:
             self.threshold = model_config["thresholds"][threshold.value]
         else:
             self.threshold = 0.5
+            logger.warning(f"Use fallback threshold value: {self.threshold}")
 
         char_set = set(model_config["char_set"])
         if len(char_set) != len(model_config["char_set"]):
@@ -80,25 +84,44 @@ class MlValidator:
 
         self.common_feature_list = []
         self.unique_feature_list = []
-        logger.info("Init ML validator with %s provider; config:'%s' md5:%s model:'%s' md5:%s", providers,
-                    ml_config_path, md5_config, ml_model_path, md5_model)
-        logger.debug("ML validator details: %s", model_config)
+        if logger.isEnabledFor(logging.INFO):
+            config_dbg = str(model_config) if logger.isEnabledFor(logging.DEBUG) else ''
+            config_md5 = hashlib.md5(__ml_config_data).hexdigest()
+            model_md5 = hashlib.md5(self.__ml_model_data).hexdigest()
+            logger.info("Init ML validator with providers: '%s' ; model:'%s' md5:%s ; config:'%s' md5:%s ; %s",
+                        self.providers, ml_config_path, config_md5, ml_model_path, model_md5, config_dbg)
         for feature_definition in model_config["features"]:
             feature_class = feature_definition["type"]
             kwargs = feature_definition.get("kwargs", {})
             feature_constructor = getattr(features, feature_class, None)
             if feature_constructor is None:
-                raise ValueError(f'Error while parsing model details. Cannot create feature "{feature_class}"')
+                raise ValueError(f"Error while parsing model details. Cannot create feature '{feature_class}'"
+                                 f" from {feature_definition}")
             try:
                 feature = feature_constructor(**kwargs)
             except TypeError:
-                raise TypeError(f'Error while parsing model details. Cannot create feature "{feature_class}"'
-                                f' with kwargs "{kwargs}"')
+                logger.error(f"Error while parsing model details. Cannot create feature '{feature_class}'"
+                             f" from {feature_definition}")
+                raise
             if feature_definition["type"] in ["RuleName"]:
                 self.unique_feature_list.append(feature)
             else:
                 self.common_feature_list.append(feature)
 
+    def __reduce__(self):
+        # TypeError: cannot pickle 'onnxruntime.capi.onnxruntime_pybind11_state.InferenceSession' object
+        self.__session = None
+        return super().__reduce__()
+
+    @property
+    def session(self) -> InferenceSession:
+        """session getter to prevent pickle error"""
+        if not self.__session:
+            self.__session = InferenceSession(self.__ml_model_data, providers=self.providers)
+        if not self.__session:
+            raise RuntimeError("InferenceSession was not initialized!")
+        return self.__session
+
     def encode(self, text: str, limit: int) -> np.ndarray:
         """Encodes prepared text to array"""
         result_array: np.ndarray = np.zeros(shape=(limit, self.num_classes), dtype=np.float32)
@@ -135,7 +158,7 @@ class MlValidator:
             "value_input": value_input.astype(np.float32),
             "feature_input": feature_input.astype(np.float32),
         }
-        result = self.model_session.run(output_names=None, input_feed=input_feed)
+        result = self.session.run(output_names=None, input_feed=input_feed)
         if result and isinstance(result[0], np.ndarray):
             return result[0]
         raise RuntimeError(f"Unexpected type {type(result[0])}")
@@ -177,8 +200,8 @@ class MlValidator:
         default_candidate = candidates[0]
         line_input = self.encode_line(default_candidate.line_data_list[0].line,
                                       default_candidate.line_data_list[0].value_start)[np.newaxis]
-        variable = ""
-        value = ""
+        variable = ''
+        value = ''
         for candidate in candidates:
             if not variable and candidate.line_data_list[0].variable:
                 variable = candidate.line_data_list[0].variable
@@ -229,7 +252,7 @@ class MlValidator:
         features_list = []
         probability: np.ndarray = np.zeros(len(group_list), dtype=np.float32)
         head = tail = 0
-        for group_key, candidates in group_list:
+        for _group_key, candidates in group_list:
             line_input, variable_input, value_input, feature_array = self.get_group_features(candidates)
             line_input_list.append(line_input)
             variable_input_list.append(variable_input)
@@ -250,8 +273,8 @@ class MlValidator:
                                                             features_list)
         is_cred = probability > self.threshold
         if logger.isEnabledFor(logging.DEBUG):
-            for i in range(len(is_cred)):
-                logger.debug("ML decision: %s with prediction: %s for value: %s", is_cred[i], probability[i],
+            for i, decision in enumerate(is_cred):
+                logger.debug("ML decision: %s with prediction: %s for value: %s", decision, probability[i],
                              group_list[i][0])
         # apply cast to float to avoid json export issue
         return is_cred, probability.astype(float)