credsweeper 1.10.7__py3-none-any.whl → 1.11.0__py3-none-any.whl

credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.10.7"
+__version__ = "1.11.0"

credsweeper/common/keyword_pattern.py

@@ -26,7 +26,7 @@ class KeywordPattern:
     string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[`'\"])))?"
     left_quote = r"(?P<value_leftquote>((?P<esq>\\{1,8})?([`'\"]|&(quot|apos);)){1,4}))?"
     # Authentication scheme ( oauth | basic | bearer | apikey ) precedes to credential
-    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey)\s)?"
+    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey|ssws|ntlm)\s)?"
     value = r"(?P<value>" \
             r"(?(value_leftquote)" \
             r"(" \

credsweeper/common/morpheme_checklist.txt

@@ -960,6 +960,7 @@ nish
 nism
 node
 non
+nope
 norm
 not
 nsive
@@ -1529,6 +1530,7 @@ warn
 watch
 wave
 way
+weak
 web
 week
 weight

credsweeper/deep_scanner/bzip2_scanner.py

@@ -29,7 +29,7 @@ class Bzip2Scanner(AbstractScanner, ABC):
             bzip2_content_provider = DataContentProvider(data=bz2.decompress(data_provider.data),
                                                          file_path=new_path,
                                                          file_type=Util.get_extension(new_path),
-                                                         info=f"{data_provider.info}|BZIP2:{new_path}")
+                                                         info=f"{data_provider.info}|BZIP2:{file_path}")
             new_limit = recursive_limit_size - len(bzip2_content_provider.data)
             bzip2_candidates = self.recursive_scan(bzip2_content_provider, depth, new_limit)
             return bzip2_candidates

credsweeper/deep_scanner/deep_scanner.py

@@ -76,17 +76,32 @@ class DeepScanner(
         return self.__scanner
 
     @staticmethod
-    def get_deep_scanners(data: bytes, file_type: str, depth: int) -> List[Any]:
-        """Returns possibly scan methods for the data depends on content"""
+    def get_deep_scanners(data: bytes, file_type: str, depth: int) -> Tuple[List[Any], List[Any]]:
+        """Returns possibly scan methods for the data depends on content and fallback scanners"""
         deep_scanners: List[Any] = []
+        fallback_scanners: List[Any] = []
         if Util.is_zip(data):
             if 0 < depth:
                 deep_scanners.append(ZipScanner)
-            # probably, there might be a docx, xlxs and so on.
+            # probably, there might be a docx, xlsx and so on.
             # It might be scanned with text representation in third-party libraries.
-            deep_scanners.append(XlsxScanner)
-            deep_scanners.append(DocxScanner)
-            deep_scanners.append(PptxScanner)
+            if file_type in (".xlsx", ".ods"):
+                deep_scanners.append(XlsxScanner)
+            else:
+                fallback_scanners.append(XlsxScanner)
+            if ".docx" == file_type:
+                deep_scanners.append(DocxScanner)
+            else:
+                fallback_scanners.append(DocxScanner)
+            if ".pptx" == file_type:
+                deep_scanners.append(PptxScanner)
+            else:
+                fallback_scanners.append(PptxScanner)
+        elif Util.is_com(data):
+            if ".xls" == file_type:
+                deep_scanners.append(XlsxScanner)
+            else:
+                fallback_scanners.append(XlsxScanner)
         elif Util.is_bzip2(data):
             if 0 < depth:
                 deep_scanners.append(Bzip2Scanner)
@@ -102,25 +117,67 @@ class DeepScanner(
             deep_scanners.append(JksScanner)
         elif Util.is_asn1(data):
             deep_scanners.append(Pkcs12Scanner)
-        elif file_type in [".eml", ".mht"]:
-            if Util.is_eml(data):
-                deep_scanners.append(EmlScanner)
-            elif Util.is_xml(data) and Util.is_html(data):
-                deep_scanners.append(HtmlScanner)
-            else:
-                deep_scanners.append(ByteScanner)
         elif Util.is_xml(data):
             if Util.is_html(data):
                 deep_scanners.append(HtmlScanner)
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
             elif Util.is_mxfile(data):
                 deep_scanners.append(MxfileScanner)
-            deep_scanners.append(XmlScanner)
-        else:
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
+            else:
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
+        elif Util.is_eml(data):
+            if ".eml" == file_type:
+                deep_scanners.append(EmlScanner)
+            else:
+                fallback_scanners.append(EmlScanner)
+            fallback_scanners.append(ByteScanner)
+        elif not Util.is_binary(data):
             if 0 < depth:
                 deep_scanners.append(EncoderScanner)
                 deep_scanners.append(LangScanner)
             deep_scanners.append(ByteScanner)
-        return deep_scanners
+        else:
+            logger.warning("Cannot apply a deep scanner for type %s", file_type)
+        return deep_scanners, fallback_scanners
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    def deep_scan_with_fallback(self, data_provider: DataContentProvider, depth: int,
+                                recursive_limit_size: int) -> List[Candidate]:
+        """Scans with deep scanners and fallback scanners if possible
+
+            Args:
+                data_provider: DataContentProvider with raw data
+                depth: maximal level of recursion
+                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
+
+            Returns: list with candidates
+
+        """
+        candidates: List[Candidate] = []
+        deep_scanners, fallback_scanners = self.get_deep_scanners(data_provider.data, data_provider.file_type, depth)
+        fallback = True
+        for scan_class in deep_scanners:
+            new_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+            if new_candidates is None:
+                # scanner did not recognise the content type
+                continue
+            augment_candidates(candidates, new_candidates)
+            # this scan is successful, so fallback is not necessary
+            fallback = False
+        if fallback:
+            for scan_class in deep_scanners:
+                fallback_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+                if fallback_candidates is None:
+                    continue
+                augment_candidates(candidates, fallback_candidates)
+                # use only first successful fallback scanner
+                break
+        return candidates
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
@@ -160,17 +217,8 @@ class DeepScanner(
                                                 file_path=content_provider.file_path,
                                                 file_type=content_provider.file_type,
                                                 info=content_provider.info or info)
-            # iterate for all possibly scanner methods WITHOUT ByteContentProvider for TextContentProvider
-            scanner_classes = self.get_deep_scanners(data, content_provider.file_type, depth)
-            fallback = True
-            for scan_class in scanner_classes:
-                if new_candidates := scan_class.data_scan(self, data_provider, depth, recursive_limit_size - len(data)):
-                    augment_candidates(candidates, new_candidates)
-                    fallback = False
-            if fallback and ByteScanner not in scanner_classes and not Util.is_binary(data):
-                # wrong assumption case
-                fallback_candidates = ByteScanner.data_scan(self, data_provider, depth, recursive_limit_size)
-                augment_candidates(candidates, fallback_candidates)
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size - len(data))
+            augment_candidates(candidates, new_candidates)
         return candidates
 
     def recursive_scan(
@@ -203,16 +251,8 @@ class DeepScanner(
                                                             FilePathExtractor.FIND_BY_EXT_RULE)
             candidates.append(dummy_candidate)
         else:
-            fallback = True
-            # iterate for all possibly scanner methods
-            scanner_classes = self.get_deep_scanners(data_provider.data, data_provider.file_type, depth)
-            for scanner_class in scanner_classes:
-                if new_candidates := scanner_class.data_scan(self, data_provider, depth, recursive_limit_size):
-                    augment_candidates(candidates, new_candidates)
-                    fallback = False
-            if fallback and ByteScanner not in scanner_classes and not Util.is_binary(data_provider.data):
-                bypass_candidates = ByteScanner.data_scan(self, data_provider, depth, recursive_limit_size)
-                augment_candidates(candidates, bypass_candidates)
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size)
+            augment_candidates(candidates, new_candidates)
 
         return candidates
 

credsweeper/deep_scanner/gzip_scanner.py

@@ -31,7 +31,7 @@ class GzipScanner(AbstractScanner, ABC):
                 gzip_content_provider = DataContentProvider(data=f.read(),
                                                             file_path=new_path,
                                                             file_type=Util.get_extension(new_path),
-                                                            info=f"{data_provider.info}|GZIP:{new_path}")
+                                                            info=f"{data_provider.info}|GZIP:{file_path}")
                 new_limit = recursive_limit_size - len(gzip_content_provider.data)
                 gzip_candidates = self.recursive_scan(gzip_content_provider, depth, new_limit)
                 return gzip_candidates

credsweeper/ml_model/features/__init__.py

@@ -6,7 +6,9 @@ from credsweeper.ml_model.features.length_of_attribute import LengthOfAttribute
 from credsweeper.ml_model.features.morpheme_dense import MorphemeDense
 from credsweeper.ml_model.features.rule_name import RuleName
 from credsweeper.ml_model.features.search_in_attribute import SearchInAttribute
-from credsweeper.ml_model.features.word_in_line import WordInLine
 from credsweeper.ml_model.features.word_in_path import WordInPath
+from credsweeper.ml_model.features.word_in_postamble import WordInPostamble
+from credsweeper.ml_model.features.word_in_preamble import WordInPreamble
+from credsweeper.ml_model.features.word_in_transition import WordInTransition
 from credsweeper.ml_model.features.word_in_value import WordInValue
 from credsweeper.ml_model.features.word_in_variable import WordInVariable

credsweeper/ml_model/features/word_in_path.py

@@ -21,8 +21,10 @@ class WordInPath(WordIn):
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         # actually there must be one path because the candidates are grouped before
-        if path := candidates[0].line_data_list[0].path:
-            posix_lower_path = Path(path).as_posix().lower()
+        if file_path := candidates[0].line_data_list[0].path:
+            path = Path(file_path)
+            # apply ./ for normalised path to detect "/src" for relative path
+            posix_lower_path = path.as_posix().lower() if path.is_absolute() else f"./{path.as_posix().lower()}"
             return self.word_in_str(posix_lower_path)
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper/ml_model/features/word_in_postamble.py

@@ -0,0 +1,32 @@
+from typing import List
+
+import numpy as np
+
+from credsweeper.common.constants import ML_HUNK
+from credsweeper.credentials import Candidate
+from credsweeper.ml_model.features.word_in import WordIn
+
+
+class WordInPostamble(WordIn):
+    """Feature is true if line contains at least one word from predefined list."""
+
+    def __init__(self, words: List[str]) -> None:
+        """Feature returns array of matching words
+
+        Args:
+            words: list of predefined words - MUST BE IN LOWER CASE
+
+        """
+        super().__init__(words)
+
+    def extract(self, candidate: Candidate) -> np.ndarray:
+        """Returns true if any words in a part of line after value"""
+        postamble_end = len(candidate.line_data_list[0].line) \
+            if len(candidate.line_data_list[0].line) < candidate.line_data_list[0].value_end + ML_HUNK \
+            else candidate.line_data_list[0].value_end + ML_HUNK
+        postamble = candidate.line_data_list[0].line[candidate.line_data_list[0].value_end:postamble_end].strip()
+
+        if postamble:
+            return self.word_in_str(postamble.lower())
+        else:
+            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper/ml_model/features/word_in_preamble.py

@@ -0,0 +1,37 @@
+from typing import List
+
+import numpy as np
+
+from credsweeper.common.constants import ML_HUNK
+from credsweeper.credentials import Candidate
+from credsweeper.ml_model.features.word_in import WordIn
+
+
+class WordInPreamble(WordIn):
+    """Feature is true if line contains at least one word from predefined list."""
+
+    def __init__(self, words: List[str]) -> None:
+        """Feature returns array of matching words
+
+        Args:
+            words: list of predefined words - MUST BE IN LOWER CASE
+
+        """
+        super().__init__(words)
+
+    def extract(self, candidate: Candidate) -> np.ndarray:
+        """Returns true if any words in line before variable or value"""
+        if 0 <= candidate.line_data_list[0].variable_start:
+            preamble_start = 0 if ML_HUNK >= candidate.line_data_list[0].variable_start \
+                else candidate.line_data_list[0].variable_start - ML_HUNK
+            preamble = candidate.line_data_list[0].line[preamble_start:candidate.line_data_list[0].
+                                                        variable_start].strip()
+        else:
+            preamble_start = 0 if ML_HUNK >= candidate.line_data_list[0].value_start \
+                else candidate.line_data_list[0].value_start - ML_HUNK
+            preamble = candidate.line_data_list[0].line[preamble_start:candidate.line_data_list[0].value_start].strip()
+
+        if preamble:
+            return self.word_in_str(preamble.lower())
+        else:
+            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper/ml_model/features/{word_in_line.py → word_in_transition.py}

@@ -2,13 +2,11 @@ from typing import List
 
 import numpy as np
 
-from credsweeper.common.constants import CHUNK_SIZE
 from credsweeper.credentials import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
-from credsweeper.utils import Util
 
 
-class WordInLine(WordIn):
+class WordInTransition(WordIn):
     """Feature is true if line contains at least one word from predefined list."""
 
     def __init__(self, words: List[str]) -> None:
@@ -21,9 +19,14 @@ class WordInLine(WordIn):
         super().__init__(words)
 
     def extract(self, candidate: Candidate) -> np.ndarray:
-        """Returns true if any words in first line"""
-        subtext = Util.subtext(candidate.line_data_list[0].line, candidate.line_data_list[0].value_start, CHUNK_SIZE)
-        if subtext:
-            return self.word_in_str(subtext.lower())
+        """Returns true if any words between variable and value"""
+        if 0 <= candidate.line_data_list[0].variable_end < candidate.line_data_list[0].value_start:
+            transition = candidate.line_data_list[0].line[candidate.line_data_list[0].variable_end:candidate.
+                                                          line_data_list[0].value_start].strip()
+        else:
+            transition = ''
+
+        if transition:
+            return self.word_in_str(transition.lower())
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper/ml_model/ml_config.json

@@ -70,6 +70,38 @@
                 "attribute": "value"
             }
         },
+        {
+            "type": "SearchInAttribute",
+            "comment": "camelStyle naming detection",
+            "kwargs": {
+                "pattern": "^[a-z][a-z]{1,16}[0-9]*([A-Z]([a-z]{1,16}[0-9]*|[0-9]{1,16})){1,8}$",
+                "attribute": "value"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "PascalStyle naming detection",
+            "kwargs": {
+                "pattern": "^([A-Z]([a-z]{1,16}[0-9]*|[0-9]{1,16})){1,8}$",
+                "attribute": "value"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "UPPERCASE naming detection",
+            "kwargs": {
+                "pattern": "^(_+[0-9]{1,16}|_*[A-Z]{1,16}[0-9]*)(_+([0-9]{1,16}|[A-Z]{1,16}[0-9]*)){1,8}_*$",
+                "attribute": "value"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "lowercase naming detection",
+            "kwargs": {
+                "pattern": "^(_+[0-9]{1,16}|_*[a-z]{1,16}[0-9]*)(_+([0-9]{1,16}|[a-z]{1,16}[0-9]*)){1,8}_*$",
+                "attribute": "value"
+            }
+        },
         {
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedPatternCheck",
@@ -82,7 +114,7 @@
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedNameCheck",
             "kwargs": {
-                "pattern": "(?i:pub(lic)?_?key)",
+                "pattern": "(?i:(filters?|pub(lic)?)_?key)",
                 "attribute": "variable"
             }
         },
@@ -90,7 +122,15 @@
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedNameCheck",
             "kwargs": {
-                "pattern": "(?i:_?id$|name$|type$)",
+                "pattern": "(?i:(id|size|name|type|manager)$)",
+                "attribute": "variable"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "PWD invocation",
+            "kwargs": {
+                "pattern": "(?i:(^\\$pwd$)|(^\\$\\{#?pwd[^}]*\\}$)|(^\\$\\(pwd\\)$)|(^`pwd`$))",
                 "attribute": "variable"
             }
         },
@@ -98,44 +138,55 @@
             "type": "WordInVariable",
             "kwargs": {
                 "words": [
-                    "/",
                     " ",
+                    "/",
                     "_at",
                     "_id",
-                    "obj",
+                    "_len",
+                    "access",
+                    "cache",
+                    "client",
+                    "control",
+                    "encrypted",
+                    "example",
+                    "expire",
+                    "fake",
                     "file",
-                    "path",
+                    "filter",
+                    "fingerprint",
                     "hash",
+                    "key",
+                    "label",
+                    "length",
+                    "manager",
+                    "mock",
                     "name",
+                    "native",
+                    "obj",
+                    "option",
+                    "p/w",
+                    "parameter",
+                    "pass",
+                    "path",
+                    "project",
+                    "public",
+                    "pw",
+                    "secret",
+                    "size",
+                    "space",
+                    "status",
+                    "sword",
+                    "temp",
                     "test",
+                    "thumbprint",
                     "time",
-                    "temp",
+                    "timestamp",
+                    "title",
+                    "token",
                     "type",
-                    "mock",
-                    "size",
                     "uniq",
-                    "fake",
-                    "view",
-                    "cache",
                     "valid",
-                    "label",
-                    "title",
-                    "access",
-                    "space",
-                    "filter",
-                    "native",
-                    "status",
-                    "expire",
-                    "client",
-                    "option",
-                    "public",
-                    "project",
-                    "control",
-                    "parameter",
-                    "encrypted",
-                    "timestamp",
-                    "thumbprint",
-                    "fingerprint"
+                    "view"
                 ]
             }
         },
@@ -144,76 +195,154 @@
             "kwargs": {
                 "words": [
                     " ",
+                    "$(",
+                    "${",
                     "(",
-                    "[",
-                    ".",
                     "->",
-                    "${",
-                    "$(",
+                    ".",
                     "...",
-                    "foo",
-                    "bar",
                     "123",
+                    "<",
+                    ">",
+                    "[",
+                    "_id",
                     "abc",
-                    "xyz",
-                    "xxx",
-                    "pwd",
-                    "passwd",
-                    "pswd",
-                    "psswd",
+                    "allow",
+                    "bar",
+                    "disable",
+                    "changeme",
+                    "example",
+                    "fake",
+                    "file",
+                    "foo",
+                    "min",
+                    "mock",
+                    "my",
+                    "nil",
                     "pass",
+                    "passwd",
                     "password",
-                    "pasword",
-                    "null",
-                    "nil",
-                    "undefined",
-                    "none",
-                    "true",
-                    "false",
-                    "example",
+                    "pswd",
                     "public",
-                    "mock",
-                    "fake",
+                    "pwd",
                     "test",
-                    "allow",
-                    "my",
-                    "file",
-                    "id"
+                    "xxx",
+                    "xyz"
                 ]
             }
         },
         {
-            "type": "WordInLine",
+            "type": "WordInPreamble",
             "kwargs": {
                 "words": [
+                    "$",
+                    "%2",
+                    "%3",
+                    "&",
+                    "&amp;",
                     "(",
-                    "[",
+                    "->",
                     ".",
-                    "$",
                     "://",
+                    "?",
                     "@",
-                    "pwd",
-                    "passwd",
-                    "pswd",
-                    "psswd",
+                    "[",
+                    "approval",
+                    "assert",
+                    "case",
+                    "circle",
+                    "equal",
+                    "example",
+                    "expect",
+                    "false",
+                    "height",
+                    "image",
+                    "line",
+                    "media",
+                    "nil",
+                    "none",
+                    "null",
                     "pass",
                     "password",
-                    "pasword",
-                    "->",
-                    "null",
-                    "nil",
+                    "path",
+                    "pwd",
+                    "sqa",
+                    "test",
+                    "true",
                     "undefined",
-                    "none",
                     "unit",
+                    "width"
+                ]
+            }
+        },
+        {
+            "type": "WordInTransition",
+            "kwargs": {
+                "words": [
+                    "%2",
+                    "%3",
+                    "&",
+                    "(",
+                    "->",
+                    ".",
+                    "?",
+                    "@",
+                    "[",
+                    "bearer",
+                    "equal",
+                    "example",
+                    "expect",
+                    "line",
+                    "media",
+                    "pass",
+                    "password",
+                    "path",
                     "test",
-                    "approval",
-                    "case",
-                    "true",
-                    "false",
+                    "unit"
+                ]
+            }
+        },
+        {
+            "type": "WordInPostamble",
+            "kwargs": {
+                "words": [
+                    "$",
+                    "%2",
+                    "%3",
+                    "&",
+                    "&amp;",
+                    "(",
+                    "->",
+                    ".",
+                    "://",
+                    "?",
+                    "@",
+                    "[",
                     "assert",
+                    "case",
+                    "circle",
                     "equal",
                     "example",
-                    "expect"
+                    "expect",
+                    "false",
+                    "height",
+                    "image",
+                    "line",
+                    "media",
+                    "nil",
+                    "none",
+                    "null",
+                    "pass",
+                    "passwd",
+                    "password",
+                    "path",
+                    "pwd",
+                    "sqa",
+                    "test",
+                    "true",
+                    "undefined",
+                    "unit",
+                    "width"
                 ]
             }
         },
@@ -221,20 +350,22 @@
             "type": "WordInPath",
             "kwargs": {
                 "words": [
-                    "/test",
-                    "/config",
-                    "/src/",
-                    "/record",
-                    "/usr/local/lib/python",
+                    "/conf",
                     "/dist-packages/",
+                    "/example",
+                    "/record",
+                    "/script",
                     "/site-packages/",
-                    "/example"
+                    "/src/",
+                    "/test",
+                    "/tool",
+                    "/usr/local/lib/python",
+                    "/assets/"
                 ]
             }
         },
         {
-            "type": "MorphemeDense",
-            "kwargs": {}
+            "type": "MorphemeDense"
         },
         {
             "type": "HasHtmlTag"
@@ -255,6 +386,7 @@
                     ".bat",
                     ".bats",
                     ".bazel",
+                    ".bin",
                     ".build",
                     ".bundle",
                     ".bzl",
@@ -337,6 +469,7 @@
                     ".nix",
                     ".nolint",
                     ".odd",
+                    ".onnx",
                     ".oracle",
                     ".pan",
                     ".patch",
@@ -396,6 +529,7 @@
                     ".ts",
                     ".tsx",
                     ".txt",
+                    ".var",
                     ".vue",
                     ".xaml",
                     ".xib",

credsweeper/rules/config.yaml

@@ -576,7 +576,7 @@
   confidence: strong
   type: pattern
   values:
-    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>xox[aboprst]\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>xox[a-z]\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
   filter_type: GeneralPattern
   required_substrings:
     - xox
@@ -1407,6 +1407,37 @@
     - code
     - doc
 
+- name: Sentry Organization Auth Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntrys_eyJ[0-9A-Za-z_-]{80,8000}=*([0-9A-Za-z_-]{32,256})?)(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+    - ValueEntropyBase64Check
+  required_substrings:
+    - sntrys_eyJ
+  target:
+    - code
+    - doc
+
+- name: Sentry User Auth Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntryu_[0-9a-f]{64})(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+  required_substrings:
+    - sntryu_
+  target:
+    - code
+    - doc
+
 - name: Discord Bot Token
   severity: high
   confidence: strong

credsweeper/scanner/scanner.py

@@ -69,6 +69,7 @@ class Scanner:
             rule_path = APP_PATH / "rules" / "config.yaml"
         rule_templates = Util.yaml_load(rule_path)
         if rule_templates and isinstance(rule_templates, list):
+            rule_names = set()
             for rule_template in rule_templates:
                 try:
                     rule = Rule(self.config, rule_template)
@@ -77,6 +78,10 @@ class Scanner:
                     raise exc
                 if not self._is_available(rule):
                     continue
+                if rule.rule_name in rule_names:
+                    raise RuntimeError(f"Duplicated rule name {rule.rule_name}")
+                else:
+                    rule_names.add(rule.rule_name)
                 if 0 < rule.min_line_len:
                     if rule.rule_type == RuleType.KEYWORD:
                         self.min_keyword_len = min(self.min_keyword_len, rule.min_line_len)
@@ -141,7 +146,7 @@ class Scanner:
             # "cache" - YAPF and pycharm formatters ...
             matched_keyword = \
                 target_line_stripped_len >= self.min_keyword_len and (  #
-                    '=' in target_line_stripped or ':' in target_line_stripped)  #
+                        '=' in target_line_stripped or ':' in target_line_stripped)  #
             matched_pem_key = \
                 target_line_stripped_len >= self.min_pem_key_len \
                 and PEM_BEGIN_PATTERN in target_line_stripped and "PRIVATE" in target_line_stripped

credsweeper/secret/config.json

@@ -12,6 +12,10 @@
             ".xlsx",
             ".docx",
             ".pptx",
+            ".xls",
+            ".odp",
+            ".ods",
+            ".odt",
             ".pdf"
         ],
         "extension": [
@@ -67,7 +71,6 @@
             ".webm",
             ".webp",
             ".woff",
-            ".xls",
             ".yuv"
         ],
         "path": [

credsweeper/utils/util.py

@@ -153,19 +153,26 @@ class Util:
         return entropy < min_entropy
 
     @staticmethod
-    def is_binary(data: bytes) -> bool:
+    def is_known(data: bytes) -> bool:
         """
         Returns true if any recognized binary format found
-        or two zeroes sequence is found which never exists in text format (UTF-8, UTF-16)
-        UTF-32 is not supported
         """
         if Util.is_zip(data) \
                 or Util.is_gzip(data) \
                 or Util.is_tar(data) \
                 or Util.is_bzip2(data) \
+                or Util.is_com(data) \
                 or Util.is_pdf(data) \
                 or Util.is_elf(data):
             return True
+        return False
+
+    @staticmethod
+    def is_binary(data: bytes) -> bool:
+        """
+        Returns True when two zeroes sequence is found which never exists in text format (UTF-8, UTF-16)
+        UTF-32 is not supported
+        """
         if 0 <= data.find(b"\0\0", 0, MAX_LINE_LENGTH):
             return True
         non_ascii_cnt = 0
@@ -224,7 +231,7 @@ class Util:
             encodings = AVAILABLE_ENCODINGS
         for encoding in encodings:
             try:
-                if binary_suggest and LATIN_1 == encoding and Util.is_binary(content):
+                if binary_suggest and LATIN_1 == encoding and (Util.is_known(content) or Util.is_binary(content)):
                     # LATIN_1 may convert data (bytes in range 0x80:0xFF are transformed)
                     # so skip this encoding when checking binaries
                     logger.warning("Binary file detected")
@@ -390,6 +397,15 @@ class Util:
                     return False
         return False
 
+    @staticmethod
+    def is_com(data: bytes) -> bool:
+        """According https://en.wikipedia.org/wiki/List_of_file_signatures"""
+        if isinstance(data, bytes) and 8 < len(data):
+            if data.startswith(b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1"):
+                # Compound File Binary Format: doc, xls, ppt, msi, msg
+                return True
+        return False
+
     @staticmethod
     def is_tar(data: bytes) -> bool:
         """According https://en.wikipedia.org/wiki/List_of_file_signatures"""
@@ -520,10 +536,10 @@ class Util:
     def is_eml(data: Union[bytes, bytearray]) -> bool:
         """According to https://datatracker.ietf.org/doc/html/rfc822 lookup the fields: Date, From, To or Subject"""
         if isinstance(data, (bytes, bytearray)):
-            if ((b"\nDate:" in data or data.startswith(b"Date:"))  #
-                    and (b"\nFrom:" in data or data.startswith(b"From:"))  #
-                    and (b"\nTo:" in data or data.startswith(b"To:")  #
-                         or b"\nSubject:" in data or data.startswith(b"Subject:"))):
+            if (b"\nDate:" in data or data.startswith(b"Date:")) \
+                    and (b"\nFrom:" in data or data.startswith(b"From:")) \
+                    and (b"\nTo:" in data or data.startswith(b"To:")) \
+                    and (b"\nSubject:" in data or data.startswith(b"Subject:")):
                 return True
         return False
 

{credsweeper-1.10.7.dist-info → credsweeper-1.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.10.7
+Version: 1.11.0
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues
@@ -26,7 +26,8 @@ Requires-Dist: humanfriendly
 Requires-Dist: lxml
 Requires-Dist: numpy<2.0.0
 Requires-Dist: odfpy
-Requires-Dist: onnxruntime
+Requires-Dist: onnxruntime; platform_system != 'Windows'
+Requires-Dist: onnxruntime==1.19.2; platform_system == 'Windows'
 Requires-Dist: openpyxl
 Requires-Dist: pandas
 Requires-Dist: pdfminer-six
@@ -37,6 +38,7 @@ Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Requires-Dist: pyyaml
 Requires-Dist: whatthepatch
+Requires-Dist: xlrd
 Description-Content-Type: text/markdown
 
 # CredSweeper

{credsweeper-1.10.7.dist-info → credsweeper-1.11.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-credsweeper/__init__.py,sha256=bsu6pMpQo7yoOF3zKYBhzmGZk1Y7W-1miVXpWCkScRc,632
+credsweeper/__init__.py,sha256=QgMuMG73C_jpq7fXbkUOXA9CHYf87zF9458ytKtAUl0,632
 credsweeper/__main__.py,sha256=jlI83ctJJfF0koMqP6u24JASC7MIPA2g1POx7aeuaQ8,17187
 credsweeper/app.py,sha256=sexUp4Qced22AhvbcVlb5C-QtJRoDmUp5qhc_nwj248,21369
 credsweeper/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -6,8 +6,8 @@ credsweeper/common/__init__.py,sha256=mYiHEDV0hSeWcFx0Wb8oIRDCPR92ben0mCuC9-gCTg
 credsweeper/common/constants.py,sha256=plBHrIVfj4CBpymIgLxTPiYr66_By3QKlgCoHYVKPLc,5534
 credsweeper/common/keyword_checklist.py,sha256=6EKNdMMryZykedAOhEc-MF1byi5oXmAiljq61T_nco4,2258
 credsweeper/common/keyword_checklist.txt,sha256=a8GW-wF6D83uVFYxMWEsUFlth6c1B_KDpF8_Xpj0mE8,7169
-credsweeper/common/keyword_pattern.py,sha256=IUvXQ1BWawzK_H8fkMJWUJBdPmi69RW5-VD36LfZNVE,2608
-credsweeper/common/morpheme_checklist.txt,sha256=yPfBCSlLCHrzMtZELZbco4PFFWgq_cVOLJF4VTA2b9M,8852
+credsweeper/common/keyword_pattern.py,sha256=JYwIrtMFzQhHSRJWudnbFLgE2i9dOEY4_EFYsgbIBzg,2618
+credsweeper/common/morpheme_checklist.txt,sha256=Q-vc60F05Y-DiZP9rYahouPRf7kxEyy9gsXQTWE0pc4,8862
 credsweeper/config/__init__.py,sha256=3_lLgF2P-EurNupTYmHvY11Ba8rdjhLJAAfTiMJW4mY,45
 credsweeper/config/config.py,sha256=Rfc8YLa4bcG_AMequbfQ-HggS74jC4OqTtaWOoGxJdA,2630
 credsweeper/credentials/__init__.py,sha256=bn7BEgGqKcwlCLi5-7_sXlBmIpK7s5RrNcG_e01L4Ck,333
@@ -20,12 +20,12 @@ credsweeper/credentials/line_data.py,sha256=gSQFDBSqypcFXApGG_WVbyWjlw7gC6rO4chG
 credsweeper/deep_scanner/__init__.py,sha256=Lp94BjQPZTgEa77E0v6xZaXZvQf2A-QTHsjqCzZxUFw,62
 credsweeper/deep_scanner/abstract_scanner.py,sha256=RLwANH7C42GMdgq_uyMH895HCUwmZFfCOTmFTzO8ni0,1404
 credsweeper/deep_scanner/byte_scanner.py,sha256=oHeA8mGe995SHqWvONhTDBIE5j50TQASHA9Mv6LHYuQ,1125
-credsweeper/deep_scanner/bzip2_scanner.py,sha256=yAMFujwA6pdUe3xpdx02CCiyeDxZTZKK4n4u7Bnqz6g,1618
-credsweeper/deep_scanner/deep_scanner.py,sha256=ig3Jy5FRA0ZqJwgP4fcY2X0f93j7VADSxNyDNTzLALM,15622
+credsweeper/deep_scanner/bzip2_scanner.py,sha256=74RsjmeuffEuxmKl04lXIZt3q_Zvxj-gLHXACqVSU_o,1619
+credsweeper/deep_scanner/deep_scanner.py,sha256=lBfXRsALmI62WRNeo6QULLVSFfv9rP4aa_fOXNYZe_g,17035
 credsweeper/deep_scanner/docx_scanner.py,sha256=t0vocPDY54KBjpmQBo53n5KvOISXkB-LlxFbfuRNLMA,4128
 credsweeper/deep_scanner/eml_scanner.py,sha256=iRLr2yvBWGktT2oXxl-haqnhJN3tglO1Mej10hFk0as,3512
 credsweeper/deep_scanner/encoder_scanner.py,sha256=qszql2a-lVuzVN_bNS2EsJ-Zxpqql52o1sJsLnpjX7M,1279
-credsweeper/deep_scanner/gzip_scanner.py,sha256=DwG3rSC2YxvE6aTiSKDXoqMSM1wCag_1AX2NYnDQYTM,1695
+credsweeper/deep_scanner/gzip_scanner.py,sha256=jIOHZ9oYBV_LIzJjY5JU6NgbGhrLg2rbOYp3X-iv4QU,1696
 credsweeper/deep_scanner/html_scanner.py,sha256=eWzEbsHTD1mZo-KDhReF7w0ZID_P_u-DMf_WLC5v3IE,1440
 credsweeper/deep_scanner/jks_scanner.py,sha256=pnsmF3XNRY0PwEtbC23qygjPhDd74XwY9iZkddOTBYo,1929
 credsweeper/deep_scanner/lang_scanner.py,sha256=dZnMPylXjnCt673vOsO-C-QAFn20RtaGb845mxjkCxs,1304
@@ -108,10 +108,10 @@ credsweeper/filters/group/weird_base64_token.py,sha256=nMAmhwMzBZ-La1pJnZHVOavPa
 credsweeper/logger/__init__.py,sha256=qoRn8hBnzjqDMSPAmavHbpsuyC0dmxuKqbO_v50EcDU,45
 credsweeper/logger/logger.py,sha256=E427IHUgX5jOCUKYlXKwQiyXNIUSYBpP45ldNCfVjFw,1828
 credsweeper/ml_model/__init__.py,sha256=Bvw_WgOmNwzFrBP0tKgWapasYcU7IRT0zMpbmWD6DwU,58
-credsweeper/ml_model/ml_config.json,sha256=HgjZicEsbXd0T6LFO9YEtUhlTqCBZ5HFig1F7yHPh5E,12078
-credsweeper/ml_model/ml_model.onnx,sha256=o50IaStZWRAk5VciLFglNYQiYimq9udGQed0dpoE5SY,9605528
+credsweeper/ml_model/ml_config.json,sha256=QDPNRl-QT9TY7NPgOjjsekDdPvxI3IZkyXcr5VSa6eY,16049
+credsweeper/ml_model/ml_model.onnx,sha256=1z_vsA2d74qvEUfDDxGGsWmUvRLotd7AM1C7QlkO1bE,10979845
 credsweeper/ml_model/ml_validator.py,sha256=uM5D9s-2r_qqlZWItWWnp9o1gx5FuJo3E-0Y6OhfqYM,11907
-credsweeper/ml_model/features/__init__.py,sha256=v63Ri6rrjKfD1yN47dA6i9tn9hpdjqK06afHJHV5-ig,858
+credsweeper/ml_model/features/__init__.py,sha256=OtWcKTbXsM4hi4P8G6f-T_FC6pQVOz_le2FbN2Kak1c,1020
 credsweeper/ml_model/features/entropy_evaluation.py,sha256=Mzv6bSui2XbX-WBLHk0gI0x_9RHgRjM7n9ep0VfDCIw,2605
 credsweeper/ml_model/features/feature.py,sha256=6qYXKh1vDC0rgFn1zrXsCbr52W4r3jL6Ju1UUECBPWY,1094
 credsweeper/ml_model/features/file_extension.py,sha256=ffPiPV9TM9axvBcnZE_2nE761rv7mNFzpQjo5SpAuRY,688
@@ -122,29 +122,31 @@ credsweeper/ml_model/features/morpheme_dense.py,sha256=k9UZiy19uVFSWpLMhfhaDZYue
 credsweeper/ml_model/features/rule_name.py,sha256=2J5vf8o-FHvKN9iYbupp2dMjOeln7XwDYaC3kNNwwJk,666
 credsweeper/ml_model/features/search_in_attribute.py,sha256=aHwUZ5iF4BUtNLa5BUi3ufEhcQWPa7647Oy5hBauwhQ,692
 credsweeper/ml_model/features/word_in.py,sha256=TE6fjP0UZHq3pbydxUFIvwN8M-V82VFbjd_a76B-l_k,1996
-credsweeper/ml_model/features/word_in_line.py,sha256=RmQgcyKE7vgduFv16mI9CUdWdW4EFPtkHHR83OjbKxM,974
-credsweeper/ml_model/features/word_in_path.py,sha256=FjjzdyPvEsIYwkB94-0bdbHOtKNyg22k6izTStNpXNk,1014
+credsweeper/ml_model/features/word_in_path.py,sha256=5jz2pRPF_wS9IIuiAOF3Ev73lXIPdyC0PQK1hrc08tk,1184
+credsweeper/ml_model/features/word_in_postamble.py,sha256=Yf6mcnZsnWxmtD5M_Zd1hb-MXyY2DvOJiFE9XcJs3ok,1192
+credsweeper/ml_model/features/word_in_preamble.py,sha256=mVA7sEaO_VfoJJ5rwwr16iK5jbZgJ1X1oHPKyVWOkZI,1540
+credsweeper/ml_model/features/word_in_transition.py,sha256=owpXQOWD4OzCXTWypYr3HOWuR7Q29uYSTW7a6JZuNV4,1144
 credsweeper/ml_model/features/word_in_value.py,sha256=35GCjCfvpWw4-MiMAZA9YcoaqwBommArSrJ4kEAi5TA,883
 credsweeper/ml_model/features/word_in_variable.py,sha256=d02c1ieUxm1pSyOE-S-HijgliMRV9kMaI0zv-UxFAp4,823
 credsweeper/rules/__init__.py,sha256=alXS8IivUs-AKKbVHiWvSjFpg1urJZLKItuFr61HHyg,40
-credsweeper/rules/config.yaml,sha256=_RkpDyad8_uJbMdxIQK9F9-B1CER7OHMTuZtKPTsSHE,39065
+credsweeper/rules/config.yaml,sha256=NFwzEqK1SjrN_mVbjA8Vzs7ECfY_uTOuBRaeXHnRVYA,39931
 credsweeper/rules/rule.py,sha256=rU6vJ4cVoeAZdUMWalRTMuAdRuYgFxeTltvSYUJRHOU,10224
 credsweeper/scanner/__init__.py,sha256=KUh1uUEgZOd12DiXV-TQP3OvByI9tsyqN1KCdw994h8,48
-credsweeper/scanner/scanner.py,sha256=eGwMJSCSqMyDzDN7Kniqeylv3rB2SuuyJmQ5QUb7hsE,9608
+credsweeper/scanner/scanner.py,sha256=7I2H2qTs5ONwLi7K5wFOqc4WZtpaQGi-MhjkThc0FpM,9846
 credsweeper/scanner/scan_type/__init__.py,sha256=JHOdDZv5bmGsj0dhIiApvjcFzQ8LYkCQdWmhxAs4lgI,288
 credsweeper/scanner/scan_type/multi_pattern.py,sha256=VUEkwry19FMdb8wlQ-znfaTGoCf9gR6Fx-i19B3kZnI,3872
 credsweeper/scanner/scan_type/pem_key_pattern.py,sha256=R_rxKS-cx79g08BOdnhHUyDU9_je3qQuPsVPqEMPB18,1501
 credsweeper/scanner/scan_type/scan_type.py,sha256=h57bUeG58HKhSapMW_25x3mdZTFQ5pToeXkHd9R17wg,9474
 credsweeper/scanner/scan_type/single_pattern.py,sha256=TCbjbeK0NrWzU6eBMZqHjYXo3xAfc0a7z3HDbbyHY-c,954
-credsweeper/secret/config.json,sha256=GLmo20U9xEJbZ8K0diprxkUYiulWIvD8-jEH8zcycAg,3237
+credsweeper/secret/config.json,sha256=nc7RZvoCz3JCrv2fiaz1hh4s3NFghBuyT21Xk9x1NiU,3297
 credsweeper/secret/log.yaml,sha256=h29atN5Kvk68oKuTYG2Mi4f2uNO3dvwhOkzCRBKo1rg,952
 credsweeper/utils/__init__.py,sha256=wPdTkrSBAkR3rppFZ68k6MiT_P7tIHuAb3AcwndJCWg,63
 credsweeper/utils/entropy_validator.py,sha256=711xCIBGAy-Pb6wqbMpEToa4dOYj5_CmkbKHygLeQrI,2796
 credsweeper/utils/hop_stat.py,sha256=0D7xB1CVAUhseOZWvLZXxn3MYHKZnfnFJ8hj7tONiyU,2978
 credsweeper/utils/pem_key_detector.py,sha256=Z1LJYm4WAqppF2ooj81-lbhrg2woiNKiMk8Nt4DV-G8,7721
-credsweeper/utils/util.py,sha256=oI4uaMAEoZUVwqPOsjWSp4boKGCQtS6dui0yFbPCsy8,29896
-credsweeper-1.10.7.dist-info/METADATA,sha256=mZ29rR4tboTGU7wO-rs-QxM96g8pIyk6qb5qWrsgaQc,10389
-credsweeper-1.10.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-credsweeper-1.10.7.dist-info/entry_points.txt,sha256=SLGNZshvi3zpWPhVmRP-oDXRMRPBS4tzRDy6xYOXwqA,58
-credsweeper-1.10.7.dist-info/licenses/LICENSE,sha256=aU7mGjBKbmRHNLVXXzcPdKmTtBxRwDPtjflQRfN7fFg,1065
-credsweeper-1.10.7.dist-info/RECORD,,
+credsweeper/utils/util.py,sha256=koK8Sat8wjWHncOy0MQbnKRUrdxrJ77gt8U6spRG-oA,30451
+credsweeper-1.11.0.dist-info/METADATA,sha256=ntR_qdmXm2ZVenmDOzzlxcaf6lFGL8xOXHf1kNo3fsg,10504
+credsweeper-1.11.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+credsweeper-1.11.0.dist-info/entry_points.txt,sha256=SLGNZshvi3zpWPhVmRP-oDXRMRPBS4tzRDy6xYOXwqA,58
+credsweeper-1.11.0.dist-info/licenses/LICENSE,sha256=aU7mGjBKbmRHNLVXXzcPdKmTtBxRwDPtjflQRfN7fFg,1065
+credsweeper-1.11.0.dist-info/RECORD,,