credsweeper 1.10.6__tar.gz → 1.10.8__tar.gz

{credsweeper-1.10.6 → credsweeper-1.10.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.10.6
+Version: 1.10.8
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues
@@ -26,7 +26,8 @@ Requires-Dist: humanfriendly
 Requires-Dist: lxml
 Requires-Dist: numpy<2.0.0
 Requires-Dist: odfpy
-Requires-Dist: onnxruntime
+Requires-Dist: onnxruntime; platform_system != 'Windows'
+Requires-Dist: onnxruntime==1.19.2; platform_system == 'Windows'
 Requires-Dist: openpyxl
 Requires-Dist: pandas
 Requires-Dist: pdfminer-six
@@ -37,6 +38,7 @@ Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Requires-Dist: pyyaml
 Requires-Dist: whatthepatch
+Requires-Dist: xlrd
 Description-Content-Type: text/markdown
 
 # CredSweeper
@@ -87,7 +89,7 @@ Full documentation can be found here: <https://credsweeper.readthedocs.io/>
 
 ### Main Requirements
 
-- Python 3.8, 3.9, 3.10, 3.11, 3.12
+- Python 3.9, 3.10, 3.11, 3.12
 
 ### Installation
 

{credsweeper-1.10.6 → credsweeper-1.10.8}/README.md

@@ -46,7 +46,7 @@ Full documentation can be found here: <https://credsweeper.readthedocs.io/>
 
 ### Main Requirements
 
-- Python 3.8, 3.9, 3.10, 3.11, 3.12
+- Python 3.9, 3.10, 3.11, 3.12
 
 ### Installation
 

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.10.6"
+__version__ = "1.10.8"

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/__main__.py

@@ -205,8 +205,8 @@ def get_arguments() -> Namespace:
                         metavar="POSITIVE_INT")
     parser.add_argument("--thrifty",
                         help="clear objects after scan to reduce memory consumption",
-                        action="store_const",
-                        const=True)
+                        action=BooleanOptionalAction,
+                        default=True)
     parser.add_argument("--skip_ignored",
                         help="parse .gitignore files and skip credentials from ignored objects",
                         dest="skip_ignored",

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/common/keyword_pattern.py

@@ -3,27 +3,30 @@ import re
 
 class KeywordPattern:
     """Pattern set of keyword types"""
-    key_left = r"(\\[nrt]|%[0-9a-f]{2})?"\
-               r"(?P<variable>(([`'\"]+[^:='\"`}<>\\/&?]*|[^:='\"`}<>\s()\\/&?;,%]*)" \
+    key_left = r"(\\[nrt]|%[0-9a-f]{2})?" \
+               r"(?P<variable>(([`'\"]{1,8}[^:='\"`}<>\\/&?]*|[^:='\"`}<>\s()\\/&?;,%]*)" \
                r"(?P<keyword>"
     # there will be inserted a keyword
     key_right = r")" \
-                r"(&(quot|apos);|[^%:='\"`<>{?!&]*)[`'\"]*))"  # <variable>
-    separator = r"(\s|\\+[tnr])*\]?(\s|\\+[tnr])*" \
-                r"(?P<separator>:( [a-z]{3,9}[?]? )?=|:|=(>|&gt;|\\u0026gt;)|!=|===|==|=|%3d)" \
-                r"(\s|\\+[tnr])*"
+                r"[^%:='\"`<>{?!&]*" \
+                r")" \
+                r"(&(quot|apos);|%[0-9a-f]{2}|[`'\"])*" \
+                r")"  # <variable>
+    separator = r"(\s|\\{1,8}[tnr])*\]?(\s|\\{1,8}[tnr])*" \
+                r"(?P<separator>:(\s[a-z]{3,9}[?]?\s)?=|:|=(>|&gt;|\\u0026gt;)|!==|!=|===|==|=|%3d)" \
+                r"(\s|\\{1,8}[tnr])*"
     # might be curly, square or parenthesis with words before
     wrap = r"(?P<wrap>(" \
-           r"(new(\s|\\+[tnr])+)?" \
+           r"(new(\s|\\{1,8}[tnr]){1,8})?" \
            r"([0-9a-z_.]|-(>|(&|\\\\*u0026)gt;))*" \
-           r"[\[\(\{]"\
-           r"(\s|\\+[tnr])*" \
+           r"[\[\(\{]" \
+           r"(\s|\\{1,8}[tnr])*" \
            r"([0-9a-z_]{1,32}=)?" \
-           r")+)?"
+           r"){1,8})?"
     string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[`'\"])))?"
     left_quote = r"(?P<value_leftquote>((?P<esq>\\{1,8})?([`'\"]|&(quot|apos);)){1,4}))?"
     # Authentication scheme ( oauth | basic | bearer | apikey ) precedes to credential
-    auth_keywords = r"( ?(oauth|bot|basic|bearer|apikey|accesskey) )?"
+    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey|ssws|ntlm)\s)?"
     value = r"(?P<value>" \
             r"(?(value_leftquote)" \
             r"(" \
@@ -31,11 +34,15 @@ class KeywordPattern:
             r"(?(esq)((?!(?P=esq)([`'\"]|&(quot|apos);)).)|((?!(?P=value_leftquote)).)))" \
             r"|" \
             r"(?!&(quot|apos);)" \
-            r"(\\+([ tnr]|[^\s`'\"])|[^\s`'\",;\\])" \
-            r"){3,8000}" \
+            r"(\\{1,8}([ tnr]|[^\s`'\"])" \
+            r"|" \
+            r"(?P<url_esc>%[0-9a-f]{2})" \
+            r"|" \
+            r"(?(url_esc)[^\s`'\",;\\&]|[^\s`'\",;\\])" \
+            r")){3,8000}" \
             r"|(\{[^}]{3,8000}\})" \
             r"|(<[^>]{3,8000}>)" \
-            r")"
+            r")"  # <value>
     right_quote = r"(?(value_leftquote)" \
                   r"(?P<value_rightquote>(?<!\\)(?P=value_leftquote)|\\$|(?<=[0-9a-z+_/-])$)" \
                   r"|" \
@@ -44,7 +51,7 @@ class KeywordPattern:
     @classmethod
     def get_keyword_pattern(cls, keyword: str) -> re.Pattern:
         """Returns compiled regex pattern"""
-        expression = "".join([  #
+        expression = ''.join([  #
             cls.key_left,  #
             keyword,  #
             cls.key_right,  #

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/deep_scanner/bzip2_scanner.py

@@ -29,7 +29,7 @@ class Bzip2Scanner(AbstractScanner, ABC):
             bzip2_content_provider = DataContentProvider(data=bz2.decompress(data_provider.data),
                                                          file_path=new_path,
                                                          file_type=Util.get_extension(new_path),
-                                                         info=f"{data_provider.info}|BZIP2:{new_path}")
+                                                         info=f"{data_provider.info}|BZIP2:{file_path}")
             new_limit = recursive_limit_size - len(bzip2_content_provider.data)
             bzip2_candidates = self.recursive_scan(bzip2_content_provider, depth, new_limit)
             return bzip2_candidates

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/deep_scanner/deep_scanner.py

@@ -76,17 +76,32 @@ class DeepScanner(
         return self.__scanner
 
     @staticmethod
-    def get_deep_scanners(data: bytes, file_type: str, depth: int) -> List[Any]:
-        """Returns possibly scan methods for the data depends on content"""
+    def get_deep_scanners(data: bytes, file_type: str, depth: int) -> Tuple[List[Any], List[Any]]:
+        """Returns possibly scan methods for the data depends on content and fallback scanners"""
         deep_scanners: List[Any] = []
+        fallback_scanners: List[Any] = []
         if Util.is_zip(data):
             if 0 < depth:
                 deep_scanners.append(ZipScanner)
-            # probably, there might be a docx, xlxs and so on.
+            # probably, there might be a docx, xlsx and so on.
             # It might be scanned with text representation in third-party libraries.
-            deep_scanners.append(XlsxScanner)
-            deep_scanners.append(DocxScanner)
-            deep_scanners.append(PptxScanner)
+            if file_type in (".xlsx", ".ods"):
+                deep_scanners.append(XlsxScanner)
+            else:
+                fallback_scanners.append(XlsxScanner)
+            if ".docx" == file_type:
+                deep_scanners.append(DocxScanner)
+            else:
+                fallback_scanners.append(DocxScanner)
+            if ".pptx" == file_type:
+                deep_scanners.append(PptxScanner)
+            else:
+                fallback_scanners.append(PptxScanner)
+        elif Util.is_com(data):
+            if ".xls" == file_type:
+                deep_scanners.append(XlsxScanner)
+            else:
+                fallback_scanners.append(XlsxScanner)
         elif Util.is_bzip2(data):
             if 0 < depth:
                 deep_scanners.append(Bzip2Scanner)
@@ -102,25 +117,67 @@ class DeepScanner(
             deep_scanners.append(JksScanner)
         elif Util.is_asn1(data):
             deep_scanners.append(Pkcs12Scanner)
-        elif file_type in [".eml", ".mht"]:
-            if Util.is_eml(data):
-                deep_scanners.append(EmlScanner)
-            elif Util.is_xml(data) and Util.is_html(data):
-                deep_scanners.append(HtmlScanner)
-            else:
-                deep_scanners.append(ByteScanner)
         elif Util.is_xml(data):
             if Util.is_html(data):
                 deep_scanners.append(HtmlScanner)
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
             elif Util.is_mxfile(data):
                 deep_scanners.append(MxfileScanner)
-            deep_scanners.append(XmlScanner)
-        else:
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
+            else:
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
+        elif Util.is_eml(data):
+            if ".eml" == file_type:
+                deep_scanners.append(EmlScanner)
+            else:
+                fallback_scanners.append(EmlScanner)
+            fallback_scanners.append(ByteScanner)
+        elif not Util.is_binary(data):
             if 0 < depth:
                 deep_scanners.append(EncoderScanner)
                 deep_scanners.append(LangScanner)
             deep_scanners.append(ByteScanner)
-        return deep_scanners
+        else:
+            logger.warning("Cannot apply a deep scanner for type %s", file_type)
+        return deep_scanners, fallback_scanners
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    def deep_scan_with_fallback(self, data_provider: DataContentProvider, depth: int,
+                                recursive_limit_size: int) -> List[Candidate]:
+        """Scans with deep scanners and fallback scanners if possible
+
+            Args:
+                data_provider: DataContentProvider with raw data
+                depth: maximal level of recursion
+                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
+
+            Returns: list with candidates
+
+        """
+        candidates: List[Candidate] = []
+        deep_scanners, fallback_scanners = self.get_deep_scanners(data_provider.data, data_provider.file_type, depth)
+        fallback = True
+        for scan_class in deep_scanners:
+            new_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+            if new_candidates is None:
+                # scanner did not recognise the content type
+                continue
+            augment_candidates(candidates, new_candidates)
+            # this scan is successful, so fallback is not necessary
+            fallback = False
+        if fallback:
+            for scan_class in deep_scanners:
+                fallback_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+                if fallback_candidates is None:
+                    continue
+                augment_candidates(candidates, fallback_candidates)
+                # use only first successful fallback scanner
+                break
+        return candidates
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
@@ -160,17 +217,8 @@ class DeepScanner(
                                                 file_path=content_provider.file_path,
                                                 file_type=content_provider.file_type,
                                                 info=content_provider.info or info)
-            # iterate for all possibly scanner methods WITHOUT ByteContentProvider for TextContentProvider
-            scanner_classes = self.get_deep_scanners(data, content_provider.file_type, depth)
-            fallback = True
-            for scan_class in scanner_classes:
-                if new_candidates := scan_class.data_scan(self, data_provider, depth, recursive_limit_size - len(data)):
-                    augment_candidates(candidates, new_candidates)
-                    fallback = False
-            if fallback and ByteScanner not in scanner_classes and not Util.is_binary(data):
-                # wrong assumption case
-                fallback_candidates = ByteScanner.data_scan(self, data_provider, depth, recursive_limit_size)
-                augment_candidates(candidates, fallback_candidates)
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size - len(data))
+            augment_candidates(candidates, new_candidates)
         return candidates
 
     def recursive_scan(
@@ -203,16 +251,8 @@ class DeepScanner(
                                                             FilePathExtractor.FIND_BY_EXT_RULE)
             candidates.append(dummy_candidate)
         else:
-            fallback = True
-            # iterate for all possibly scanner methods
-            scanner_classes = self.get_deep_scanners(data_provider.data, data_provider.file_type, depth)
-            for scanner_class in scanner_classes:
-                if new_candidates := scanner_class.data_scan(self, data_provider, depth, recursive_limit_size):
-                    augment_candidates(candidates, new_candidates)
-                    fallback = False
-            if fallback and ByteScanner not in scanner_classes and not Util.is_binary(data_provider.data):
-                bypass_candidates = ByteScanner.data_scan(self, data_provider, depth, recursive_limit_size)
-                augment_candidates(candidates, bypass_candidates)
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size)
+            augment_candidates(candidates, new_candidates)
 
         return candidates
 

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/deep_scanner/gzip_scanner.py

@@ -31,7 +31,7 @@ class GzipScanner(AbstractScanner, ABC):
                 gzip_content_provider = DataContentProvider(data=f.read(),
                                                             file_path=new_path,
                                                             file_type=Util.get_extension(new_path),
-                                                            info=f"{data_provider.info}|GZIP:{new_path}")
+                                                            info=f"{data_provider.info}|GZIP:{file_path}")
                 new_limit = recursive_limit_size - len(gzip_content_provider.data)
                 gzip_candidates = self.recursive_scan(gzip_content_provider, depth, new_limit)
                 return gzip_candidates

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/filters/value_allowlist_check.py

@@ -8,22 +8,35 @@ from credsweeper.utils import Util
 
 
 class ValueAllowlistCheck(Filter):
-    """Check that patterns from the list is not present in the candidate value."""
+    """Check that the patterns do not MATCH the candidate value."""
 
     ALLOWED = [
         r"ENC\(.*\)",  #
         r"ENC\[.*\]",  #
         r"\$\{(\*|[0-9]+|[a-z_].*)\}",  #
-        r"\$([0-9]+\b|[a-z_]+[0-9a-z_]*)",  #
+        r"\$[0-9]+(\s|$)",  #
         r"\$\$[a-z_]+(\^%[0-9a-z_]+)?",  #
-        r"#\{.*\}",  #
+        r"#\{.+\}",  # Ruby: String Interpolation
         r"\{\{.+\}\}",  #
-        r"\S{0,5}\*{5,}",  #
         r".*@@@hl@@@(암호|비번|PW|PASS)@@@endhl@@@",  #
     ]
 
     ALLOWED_PATTERN = re.compile(Util.get_regex_combine_or(ALLOWED), flags=re.IGNORECASE)
-    ALLOWED_UNQUOTED_PATTERN = re.compile(r"[~a-z0-9_]+((\.|->)[a-z0-9_]+)+\(.*$", flags=re.IGNORECASE)
+
+    ALLOWED_QUOTED = [
+        r"\$[a-z_]+[0-9a-z_]*([$\s]|$)",  #
+        r".*\*\*\*",  #
+    ]
+
+    ALLOWED_QUOTED_PATTERN = re.compile(Util.get_regex_combine_or(ALLOWED_QUOTED), flags=re.IGNORECASE)
+
+    ALLOWED_UNQUOTED = [
+        r"[~a-z0-9_]+((\.|->)[a-z0-9_]+)+\(.*$",  #
+        r"\$[a-z_]+[0-9a-z_]*\b",  #
+        r".*\*\*\*\*\*",  #
+    ]
+
+    ALLOWED_UNQUOTED_PATTERN = re.compile(Util.get_regex_combine_or(ALLOWED_UNQUOTED), flags=re.IGNORECASE)
 
     def __init__(self, config: Config = None) -> None:
         pass
@@ -42,8 +55,11 @@ class ValueAllowlistCheck(Filter):
 
         if self.ALLOWED_PATTERN.match(line_data.value):
             return True
-
-        if not line_data.is_well_quoted_value and self.ALLOWED_UNQUOTED_PATTERN.match(line_data.value):
-            return True
+        elif line_data.is_well_quoted_value:
+            if self.ALLOWED_QUOTED_PATTERN.match(line_data.value):
+                return True
+        else:
+            if self.ALLOWED_UNQUOTED_PATTERN.match(line_data.value):
+                return True
 
         return False

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/filters/value_blocklist_check.py

@@ -11,8 +11,11 @@ class ValueBlocklistCheck(Filter):
         "true",
         "false",
         "null",
+        "none",
         "bearer",
         "string",
+        "value",
+        "undefined",
     ]
 
     def __init__(self, config: Config = None) -> None:

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/ml_model/features/word_in_path.py

@@ -21,8 +21,10 @@ class WordInPath(WordIn):
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         # actually there must be one path because the candidates are grouped before
-        if path := candidates[0].line_data_list[0].path:
-            posix_lower_path = Path(path).as_posix().lower()
+        if file_path := candidates[0].line_data_list[0].path:
+            path = Path(file_path)
+            # apply ./ for normalised path to detect "/src" for relative path
+            posix_lower_path = path.as_posix().lower() if path.is_absolute() else f"./{path.as_posix().lower()}"
             return self.word_in_str(posix_lower_path)
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/rules/config.yaml

@@ -576,7 +576,7 @@
   confidence: strong
   type: pattern
   values:
-    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>xox[aboprst]\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>xox[a-z]\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
   filter_type: GeneralPattern
   required_substrings:
     - xox
@@ -1391,6 +1391,53 @@
     - code
     - doc
 
+- name: Tavily API Key
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>tvly-[0-9A-Za-z_-]{32,40})(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+    - ValueEntropyBase64Check
+  required_substrings:
+    - tvly-
+  target:
+    - code
+    - doc
+
+- name: Sentry Organization Auth Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntrys_eyJ[0-9A-Za-z_-]{80,8000}=*([0-9A-Za-z_-]{32,256})?)(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+    - ValueEntropyBase64Check
+  required_substrings:
+    - sntrys_eyJ
+  target:
+    - code
+    - doc
+
+- name: Sentry User Auth Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntryu_[0-9a-f]{64})(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+  required_substrings:
+    - sntryu_
+  target:
+    - code
+    - doc
+
 - name: Discord Bot Token
   severity: high
   confidence: strong

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/scanner/scanner.py

@@ -69,6 +69,7 @@ class Scanner:
             rule_path = APP_PATH / "rules" / "config.yaml"
         rule_templates = Util.yaml_load(rule_path)
         if rule_templates and isinstance(rule_templates, list):
+            rule_names = set()
             for rule_template in rule_templates:
                 try:
                     rule = Rule(self.config, rule_template)
@@ -77,6 +78,10 @@ class Scanner:
                     raise exc
                 if not self._is_available(rule):
                     continue
+                if rule.rule_name in rule_names:
+                    raise RuntimeError(f"Duplicated rule name {rule.rule_name}")
+                else:
+                    rule_names.add(rule.rule_name)
                 if 0 < rule.min_line_len:
                     if rule.rule_type == RuleType.KEYWORD:
                         self.min_keyword_len = min(self.min_keyword_len, rule.min_line_len)
@@ -141,7 +146,7 @@ class Scanner:
             # "cache" - YAPF and pycharm formatters ...
             matched_keyword = \
                 target_line_stripped_len >= self.min_keyword_len and (  #
-                    '=' in target_line_stripped or ':' in target_line_stripped)  #
+                        '=' in target_line_stripped or ':' in target_line_stripped)  #
             matched_pem_key = \
                 target_line_stripped_len >= self.min_pem_key_len \
                 and PEM_BEGIN_PATTERN in target_line_stripped and "PRIVATE" in target_line_stripped

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/secret/config.json

@@ -12,6 +12,10 @@
             ".xlsx",
             ".docx",
             ".pptx",
+            ".xls",
+            ".odp",
+            ".ods",
+            ".odt",
             ".pdf"
         ],
         "extension": [
@@ -67,7 +71,6 @@
             ".webm",
             ".webp",
             ".woff",
-            ".xls",
             ".yuv"
         ],
         "path": [

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/utils/hop_stat.py

@@ -25,11 +25,37 @@ class HopStat:
         ')': '0',
         '_': '-',
         '+': '=',
+        'Q': 'q',
+        'W': 'w',
+        'E': 'e',
+        'R': 'r',
+        'T': 't',
+        'Y': 'y',
+        'U': 'u',
+        'I': 'i',
+        'O': 'o',
+        'P': 'p',
         '{': '[',
         '}': ']',
         '|': '\\',
+        'A': 'a',
+        'S': 's',
+        'D': 'd',
+        'F': 'f',
+        'G': 'g',
+        'H': 'h',
+        'J': 'j',
+        'K': 'k',
+        'L': 'l',
         ':': ';',
         '"': "'",
+        'Z': 'z',
+        'X': 'x',
+        'C': 'c',
+        'V': 'v',
+        'B': 'b',
+        'N': 'n',
+        'M': 'm',
         '<': ',',
         '>': '.',
         '?': '/',
@@ -75,7 +101,7 @@ class HopStat:
 
         """
         hops = []
-        value = value.lower().translate(HopStat.TRANSLATION)
+        value = value.translate(HopStat.TRANSLATION)
         for a, b in zip(value[:-1], value[1:]):
             hop = self.__hop_dict.get((a, b))
             if hop is None:

{credsweeper-1.10.6 → credsweeper-1.10.8}/credsweeper/utils/util.py

@@ -153,19 +153,26 @@ class Util:
         return entropy < min_entropy
 
     @staticmethod
-    def is_binary(data: bytes) -> bool:
+    def is_known(data: bytes) -> bool:
         """
         Returns true if any recognized binary format found
-        or two zeroes sequence is found which never exists in text format (UTF-8, UTF-16)
-        UTF-32 is not supported
         """
         if Util.is_zip(data) \
                 or Util.is_gzip(data) \
                 or Util.is_tar(data) \
                 or Util.is_bzip2(data) \
+                or Util.is_com(data) \
                 or Util.is_pdf(data) \
                 or Util.is_elf(data):
             return True
+        return False
+
+    @staticmethod
+    def is_binary(data: bytes) -> bool:
+        """
+        Returns True when two zeroes sequence is found which never exists in text format (UTF-8, UTF-16)
+        UTF-32 is not supported
+        """
         if 0 <= data.find(b"\0\0", 0, MAX_LINE_LENGTH):
             return True
         non_ascii_cnt = 0
@@ -224,7 +231,7 @@ class Util:
             encodings = AVAILABLE_ENCODINGS
         for encoding in encodings:
             try:
-                if binary_suggest and LATIN_1 == encoding and Util.is_binary(content):
+                if binary_suggest and LATIN_1 == encoding and (Util.is_known(content) or Util.is_binary(content)):
                     # LATIN_1 may convert data (bytes in range 0x80:0xFF are transformed)
                     # so skip this encoding when checking binaries
                     logger.warning("Binary file detected")
@@ -390,6 +397,15 @@ class Util:
                     return False
         return False
 
+    @staticmethod
+    def is_com(data: bytes) -> bool:
+        """According https://en.wikipedia.org/wiki/List_of_file_signatures"""
+        if isinstance(data, bytes) and 8 < len(data):
+            if data.startswith(b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1"):
+                # Compound File Binary Format: doc, xls, ppt, msi, msg
+                return True
+        return False
+
     @staticmethod
     def is_tar(data: bytes) -> bool:
         """According https://en.wikipedia.org/wiki/List_of_file_signatures"""
@@ -520,10 +536,10 @@ class Util:
     def is_eml(data: Union[bytes, bytearray]) -> bool:
         """According to https://datatracker.ietf.org/doc/html/rfc822 lookup the fields: Date, From, To or Subject"""
         if isinstance(data, (bytes, bytearray)):
-            if ((b"\nDate:" in data or data.startswith(b"Date:"))  #
-                    and (b"\nFrom:" in data or data.startswith(b"From:"))  #
-                    and (b"\nTo:" in data or data.startswith(b"To:")  #
-                         or b"\nSubject:" in data or data.startswith(b"Subject:"))):
+            if (b"\nDate:" in data or data.startswith(b"Date:")) \
+                    and (b"\nFrom:" in data or data.startswith(b"From:")) \
+                    and (b"\nTo:" in data or data.startswith(b"To:")) \
+                    and (b"\nSubject:" in data or data.startswith(b"Subject:")):
                 return True
         return False
 

{credsweeper-1.10.6 → credsweeper-1.10.8}/pyproject.toml

@@ -14,7 +14,8 @@ dependencies = [
     "lxml",
     "numpy<2.0.0",
     "odfpy",
-    "onnxruntime",
+    "onnxruntime==1.19.2; platform_system == 'Windows'",  # Python 3.9 limitation
+    "onnxruntime; platform_system != 'Windows'",
     "openpyxl",
     "pandas",
     "pdfminer.six",
@@ -25,6 +26,7 @@ dependencies = [
     "python-pptx",
     "PyYAML",
     "whatthepatch",
+    "xlrd",
 ]
 requires-python = ">=3.9"
 readme = "README.md"