credsweeper 1.11.3__py3-none-any.whl → 1.11.5__py3-none-any.whl

credsweeper/ml_model/ml_validator.py

@@ -1,10 +1,11 @@
 import hashlib
+import json
 import logging
 from pathlib import Path
 from typing import List, Tuple, Union, Optional, Dict
 
 import numpy as np
-import onnxruntime as ort
+from onnxruntime import InferenceSession
 
 import credsweeper.ml_model.features as features
 from credsweeper.common.constants import ThresholdPreset, ML_HUNK
@@ -22,6 +23,8 @@ class MlValidator:
     # applied for unknown characters
     FAKE_CHAR = '\x01'
 
+    _dir_path = Path(__file__).parent
+
     def __init__(
             self,  #
             threshold: Union[float, ThresholdPreset],  #
@@ -36,35 +39,36 @@ class MlValidator:
             ml_model: path to ml model
             ml_providers: coma separated list of providers https://onnxruntime.ai/docs/execution-providers/
         """
-        dir_path = Path(__file__).parent
+        self.__session: Optional[InferenceSession] = None
 
         if ml_config:
             ml_config_path = Path(ml_config)
         else:
-            ml_config_path = dir_path / "ml_config.json"
+            ml_config_path = MlValidator._dir_path / "ml_config.json"
         with open(ml_config_path, "rb") as f:
-            md5_config = hashlib.md5(f.read()).hexdigest()
+            __ml_config_data = f.read()
+
+        model_config = json.loads(__ml_config_data)
 
         if ml_model:
             ml_model_path = Path(ml_model)
         else:
-            ml_model_path = dir_path / "ml_model.onnx"
+            ml_model_path = MlValidator._dir_path / "ml_model.onnx"
         with open(ml_model_path, "rb") as f:
-            md5_model = hashlib.md5(f.read()).hexdigest()
+            self.__ml_model_data = f.read()
 
         if ml_providers:
-            providers = ml_providers.split(',')
+            self.providers = ml_providers.split(',')
         else:
-            providers = ["CPUExecutionProvider"]
-        self.model_session = ort.InferenceSession(ml_model_path, providers=providers)
+            self.providers = ["CPUExecutionProvider"]
 
-        model_config = Util.json_load(ml_config_path)
         if isinstance(threshold, float):
             self.threshold = threshold
         elif isinstance(threshold, ThresholdPreset) and "thresholds" in model_config:
             self.threshold = model_config["thresholds"][threshold.value]
         else:
             self.threshold = 0.5
+            logger.warning(f"Use fallback threshold value: {self.threshold}")
 
         char_set = set(model_config["char_set"])
         if len(char_set) != len(model_config["char_set"]):
@@ -80,26 +84,44 @@ class MlValidator:
 
         self.common_feature_list = []
         self.unique_feature_list = []
-        logger.info("Init ML validator with %s provider; config:'%s' md5:%s model:'%s' md5:%s", providers,
-                    ml_config_path, md5_config, ml_model_path, md5_model)
-        logger.debug("ML validator details: %s", model_config)
+        if logger.isEnabledFor(logging.INFO):
+            config_dbg = str(model_config) if logger.isEnabledFor(logging.DEBUG) else ''
+            config_md5 = hashlib.md5(__ml_config_data).hexdigest()
+            model_md5 = hashlib.md5(self.__ml_model_data).hexdigest()
+            logger.info("Init ML validator with providers: '%s' ; model:'%s' md5:%s ; config:'%s' md5:%s ; %s",
+                        self.providers, ml_config_path, config_md5, ml_model_path, model_md5, config_dbg)
         for feature_definition in model_config["features"]:
             feature_class = feature_definition["type"]
             kwargs = feature_definition.get("kwargs", {})
             feature_constructor = getattr(features, feature_class, None)
             if feature_constructor is None:
-                raise ValueError(f'Error while parsing model details. Cannot create feature "{feature_class}"')
+                raise ValueError(f"Error while parsing model details. Cannot create feature '{feature_class}'"
+                                 f" from {feature_definition}")
             try:
                 feature = feature_constructor(**kwargs)
             except TypeError:
-                logger.error(f'Error while parsing model details. Cannot create feature "{feature_class}"'
-                             f' with kwargs "{kwargs}"')
+                logger.error(f"Error while parsing model details. Cannot create feature '{feature_class}'"
+                             f" from {feature_definition}")
                 raise
             if feature_definition["type"] in ["RuleName"]:
                 self.unique_feature_list.append(feature)
             else:
                 self.common_feature_list.append(feature)
 
+    def __reduce__(self):
+        # TypeError: cannot pickle 'onnxruntime.capi.onnxruntime_pybind11_state.InferenceSession' object
+        self.__session = None
+        return super().__reduce__()
+
+    @property
+    def session(self) -> InferenceSession:
+        """session getter to prevent pickle error"""
+        if not self.__session:
+            self.__session = InferenceSession(self.__ml_model_data, providers=self.providers)
+        if not self.__session:
+            raise RuntimeError("InferenceSession was not initialized!")
+        return self.__session
+
     def encode(self, text: str, limit: int) -> np.ndarray:
         """Encodes prepared text to array"""
         result_array: np.ndarray = np.zeros(shape=(limit, self.num_classes), dtype=np.float32)
@@ -136,7 +158,7 @@ class MlValidator:
             "value_input": value_input.astype(np.float32),
             "feature_input": feature_input.astype(np.float32),
         }
-        result = self.model_session.run(output_names=None, input_feed=input_feed)
+        result = self.session.run(output_names=None, input_feed=input_feed)
         if result and isinstance(result[0], np.ndarray):
             return result[0]
         raise RuntimeError(f"Unexpected type {type(result[0])}")
@@ -178,8 +200,8 @@ class MlValidator:
         default_candidate = candidates[0]
         line_input = self.encode_line(default_candidate.line_data_list[0].line,
                                       default_candidate.line_data_list[0].value_start)[np.newaxis]
-        variable = ""
-        value = ""
+        variable = ''
+        value = ''
         for candidate in candidates:
             if not variable and candidate.line_data_list[0].variable:
                 variable = candidate.line_data_list[0].variable
@@ -251,8 +273,8 @@ class MlValidator:
                                                             features_list)
         is_cred = probability > self.threshold
         if logger.isEnabledFor(logging.DEBUG):
-            for i in range(len(is_cred)):
-                logger.debug("ML decision: %s with prediction: %s for value: %s", is_cred[i], probability[i],
+            for i, decision in enumerate(is_cred):
+                logger.debug("ML decision: %s with prediction: %s for value: %s", decision, probability[i],
                              group_list[i][0])
         # apply cast to float to avoid json export issue
         return is_cred, probability.astype(float)

credsweeper/rules/config.yaml

@@ -3,7 +3,7 @@
   confidence: weak
   type: pattern
   values:
-    - (?P<variable>(\w*(?i:비밀번호|비번|패스워드|키|암호화?|토큰|(?<!by)pass(?!ed|ing|es|age)|\bpwd?\b|token|secret|key|cred)\w*)\s*(설정은|[=:!]{1,3}))?\s*([._0-9A-Za-z\[\]]*get(env)?\s*\(\s*(?(variable)[^,]+)|[\"'\\]*(\\*(['\"]|&(quot|apos);)){0,4}(\w*(?i:(?<!by)pass(?!ed|ing|es|age|\s+[a-z]{3,80})|\bpwd?\b|token|secret|key|cred)\w*)(\\*(['\"]|&(quot|apos);)){0,4})\s*,\s*(default\s*=\s*)?([brufl@]{1,2}(?=\\*['\"&]))?(?P<lq>(\\*(['\"]|&(quot|apos);)){1,4})(?P<value>(.(?!(?P=lq))){4,80}.?)
+    - (?P<variable>(\w*(?i:비밀번호|비번|패스워드|키|암호화?|토큰|(?<!by)pass(?!ed|ing|ion|es|age)|\bpwd?\b|token|secret|key|cred)\w*)\s*(설정은|[=:!]{1,3}))?\s*([._0-9A-Za-z\[\]]*get(env)?\s*\(\s*(?(variable)[^,]+)|[\"'\\]*(\\*(['\"]|&(quot|apos);)){0,4}(\w*(?i:(?<!by)pass(?!ed|ing|ion|es|age|\s+[a-z]{3,80})|\bpwd?\b|token|secret|key|cred)\w*)(\\*(['\"]|&(quot|apos);)){0,4})\s*,\s*(default\s*=\s*)?([brufl@]{1,2}(?=\\*['\"&]))?(?P<lq>(\\*(['\"]|&(quot|apos);)){1,4})(?P<value>(.(?!(?P=lq))){4,80}.?)
   filter_type:
     - ValueAllowlistCheck
     - LineGitBinaryCheck
@@ -34,7 +34,7 @@
   confidence: weak
   type: pattern
   values:
-    - (?P<wrap>[`'\"(])?\s*(?P<variable>(\w*(?i:(?<!by)passw?o?r?d?s?(?!ed|ing|es|age)|pwd?\b|\bp/w\b|token|secret|key|credential)\w*|비밀번호|비번|패스워드|키|암호화?|토큰))[`'\"]*(\s+(?i:is|are|was|were)(\s*[:-])?\s+|\s*(설정은|[=:!]{1,3})\s*)(?P<quote>[`'\"]{1,6})?(?P<value>(?(quote)(?(wrap)[^`'\")]{4,80}|[^`'\"]{4,80})|(?(wrap)[^`'\")]{4,80}|\S{4,80})))
+    - (?P<wrap>[`'\"(])?\s*(?P<variable>(\w*(?i:(?<!by)passw?o?r?d?s?(?!ed|ing|ion|es|age)|pwd?\b|\bp/w\b|token|secret|key|credential)\w*|비밀번호|비번|패스워드|키|암호화?|토큰))[`'\"]*(\s+(?i:is|are|was|were)(\s*[:-])?\s+|\s*(설정은|[=:!]{1,3})\s*)(?P<quote>[`'\"]{1,6})?(?P<value>(?(quote)(?(wrap)[^`'\")]{4,80}|[^`'\"]{4,80})|(?(wrap)[^`'\")]{4,80}|\S{4,80})))
   filter_type:
     - ValueAllowlistCheck
     - LineGitBinaryCheck
@@ -375,16 +375,16 @@
     - code
     - doc
 
-- name: Heroku API Key
+- name: Heroku Credentials
   severity: high
-  confidence: moderate
+  confidence: strong
   type: pattern
   values:
-    - (?i)(?P<value>heroku(.{0,20})?[0-9a-f]{8}(-[0-9a-f]{4})+-[0-9a-f]{12})(?![0-9A-Za-z_-])
+    - (?P<value>HRKU-([0-9A-Za-z_-]{60}|[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}))
   filter_type: GeneralPattern
   required_substrings:
-    - heroku
-  min_line_len: 24
+    - HRKU-
+  min_line_len: 41
   target:
     - code
     - doc
@@ -413,7 +413,49 @@
     - ValueJsonWebTokenCheck
   required_substrings:
     - eyJ
-  min_line_len: 18
+  min_line_len: 64
+  target:
+    - code
+    - doc
+
+- name: JSON Web Key
+  severity: medium
+  confidence: strong
+  type: pattern
+  values:
+    - (?P<value>\b(e(yJ|yAi|woi|wog|w0K)|W(yJ|3si|wp7|wog|w0K|3sK))[0-9A-Za-z_+/-]{60,8000})
+  filter_type:
+    - ValueJsonWebKeyCheck
+  required_substrings:
+    - eyJ
+    - eyAi
+    - ewoi
+    - ewog
+    - ew0K
+    - WyJ
+    - W3si
+    - Wwp7
+    - Wwog
+    - Ww0K
+    - W3sK
+  min_line_len: 64
+  target:
+    - code
+    - doc
+
+- name: JWK
+  severity: medium
+  confidence: moderate
+  type: multi
+  values:
+    - (?P<value>['"]?\b(?P<variable>kty)[^0-9A-Za-z_-]{1,8}(RSA|EC|oct)\b['"]?)
+    - (?P<variable>\b[dk])[^0-9A-Za-z_-]{1,8}(?P<value>[0-9A-Za-z_-]{22,8000})(?![=0-9A-Za-z_-])
+  filter_type:
+    - ValuePatternCheck
+    - ValueCoupleKeywordCheck(3)
+  required_substrings:
+    - kty
+  min_line_len: 8
   target:
     - code
     - doc
@@ -1481,7 +1523,7 @@
   confidence: moderate
   type: keyword
   values:
-    - (?<!by)pass(?!ed|ing|es|age|\s+[a-z]{3,80})|pw(d|\b)
+    - (?<!by)pass(?!ed|ing|ion|es|age|\s+[a-z]{3,80})|pw(d|\b)
   filter_type: PasswordKeyword
   use_ml: true
   min_line_len: 10

credsweeper/rules/rule.py

@@ -179,7 +179,6 @@ class Rule:
             for value in _values:
                 _pattern = KeywordPattern.get_keyword_pattern(value)
                 _patterns.append(_pattern)
-            return _patterns
         elif RuleType.MULTI == self.rule_type and 2 == len(_values) \
                 or self.rule_type in (RuleType.PATTERN, RuleType.PEM_KEY) and 0 < len(_values):
             for value in _values:
@@ -188,8 +187,9 @@ class Rule:
                 logger.warning(f"Rule {self.rule_name} has extra patterns. Only single pattern supported.")
             elif RuleType.MULTI == self.rule_type and 2 < len(_values):
                 logger.warning(f"Rule {self.rule_name} has extra patterns. Only two patterns supported.")
-            return _patterns
-        raise ValueError(f"Malformed rule config file. Rule '{self.rule_name}' type '{self.rule_type}' is invalid.")
+        else:
+            raise ValueError(f"Malformed rule config file. Rule '{self.rule_name}' type '{self.rule_type}' is invalid.")
+        return _patterns
 
     @cached_property
     def patterns(self) -> List[re.Pattern]:

credsweeper/scanner/scan_type/multi_pattern.py

@@ -37,8 +37,7 @@ class MultiPattern(ScanType):
             "Rules provided to MultiPattern.run should have pattern_type equal to MULTI_PATTERN"
 
         candidates = cls._get_candidates(config, rule, target)
-        if not candidates:
-            return candidates
+
         for candidate in candidates:
             line_pos_margin = 1
             while line_pos_margin <= cls.MAX_SEARCH_MARGIN:

credsweeper/secret/config.json

@@ -5,9 +5,13 @@
             ".aar",
             ".apk",
             ".bz2",
+            ".class",
             ".gz",
+            ".jar",
             ".lzma",
+            ".rpm",
             ".tar",
+            ".war",
             ".xz",
             ".zip"
         ],
@@ -28,7 +32,6 @@
             ".avi",
             ".bin",
             ".bmp",
-            ".class",
             ".css",
             ".dmg",
             ".ear",
@@ -40,7 +43,6 @@
             ".ico",
             ".img",
             ".info",
-            ".jar",
             ".jpeg",
             ".jpg",
             ".map",
@@ -62,10 +64,8 @@
             ".rar",
             ".rc",
             ".rc2",
-            ".rar",
             ".realm",
             ".res",
-            ".rpm",
             ".s7z",
             ".scss",
             ".so",
@@ -76,7 +76,6 @@
             ".ttf",
             ".vcxproj",
             ".vdproj",
-            ".war",
             ".wav",
             ".webm",
             ".webp",
@@ -161,7 +160,8 @@
     "bruteforce_list": [
         "",
         "changeit",
-        "changeme"
+        "changeme",
+        "tizen"
     ],
     "check_for_literals": true,
     "min_pattern_value_length": 12,

credsweeper/utils/hop_stat.py

@@ -1,5 +1,5 @@
 import statistics
-from typing import Tuple
+from typing import Tuple, Dict
 
 
 class HopStat:
@@ -62,7 +62,7 @@ class HopStat:
     })
 
     def __init__(self):
-        self.__hop_dict = dict()
+        self.__hop_dict: Dict[Tuple[str, str], int] = {}
         base = ''.join(x for x in HopStat.KEYBOARD)
         for a in (x for x in base if '\0' != x):
             for b in (x for x in base if '\0' != x):
@@ -81,7 +81,7 @@ class HopStat:
     def __get_xyz(c: str) -> Tuple[int, int, int]:
         """Returns axial coordinates of a char on keyboad qwerty"""
         x = y = z = 0
-        for i in range(len(HopStat.KEYBOARD)):
+        for i, _ in enumerate(HopStat.KEYBOARD):
             x = HopStat.KEYBOARD[i].find(c)
             if 0 <= x:
                 z = i

credsweeper/utils/pem_key_detector.py

@@ -4,7 +4,7 @@ import re
 import string
 from typing import List
 
-from credsweeper.common.constants import PEM_BEGIN_PATTERN, PEM_END_PATTERN, ENTROPY_LIMIT_BASE64
+from credsweeper.common.constants import PEM_BEGIN_PATTERN, PEM_END_PATTERN, Chars
 from credsweeper.config import Config
 from credsweeper.credentials import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
@@ -12,10 +12,12 @@ from credsweeper.utils import Util
 
 logger = logging.getLogger(__name__)
 
+ENTROPY_LIMIT_BASE64 = 4.5
+
 
 class PemKeyDetector:
     """Class to detect PEM PRIVATE keys only"""
-    base64set = set(string.ascii_uppercase) | set(string.ascii_lowercase) | set(string.digits) | {'+', '/', '='}
+    base64set = set(Chars.BASE64STDPAD_CHARS.value)
 
     ignore_starts = [PEM_BEGIN_PATTERN, "Proc-Type", "Version", "DEK-Info"]
     wrap_characters = "\\'\";,[]#*!"
@@ -64,7 +66,7 @@ class PemKeyDetector:
                     if PEM_BEGIN_PATTERN in subline:
                         begin_pattern_not_passed = False
                     continue
-                elif PEM_END_PATTERN in subline:
+                if PEM_END_PATTERN in subline:
                     if "PGP" in target.line_strip:
                         # Check if entropy is high enough for base64 set with padding sign
                         entropy = Util.get_shannon_entropy(key_data)
@@ -124,7 +126,7 @@ class PemKeyDetector:
         line = line.strip(string.whitespace)
         if line.startswith("//"):
             # simplify first condition for speed-up of doxygen style processing
-            if line.startswith("// ") or line.startswith("/// "):
+            if line.startswith(("// ", "/// ")):
                 # Assume that the commented line is to be separated from base64 code, it may be a part of PEM, otherwise
                 line = line[3:]
         if line.startswith("/*"):