credsweeper 1.10.7__tar.gz → 1.10.8__tar.gz

{credsweeper-1.10.7 → credsweeper-1.10.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.10.7
+Version: 1.10.8
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues
@@ -26,7 +26,8 @@ Requires-Dist: humanfriendly
 Requires-Dist: lxml
 Requires-Dist: numpy<2.0.0
 Requires-Dist: odfpy
-Requires-Dist: onnxruntime
+Requires-Dist: onnxruntime; platform_system != 'Windows'
+Requires-Dist: onnxruntime==1.19.2; platform_system == 'Windows'
 Requires-Dist: openpyxl
 Requires-Dist: pandas
 Requires-Dist: pdfminer-six
@@ -37,6 +38,7 @@ Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Requires-Dist: pyyaml
 Requires-Dist: whatthepatch
+Requires-Dist: xlrd
 Description-Content-Type: text/markdown
 
 # CredSweeper

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.10.7"
+__version__ = "1.10.8"

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/common/keyword_pattern.py

@@ -26,7 +26,7 @@ class KeywordPattern:
     string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[`'\"])))?"
     left_quote = r"(?P<value_leftquote>((?P<esq>\\{1,8})?([`'\"]|&(quot|apos);)){1,4}))?"
     # Authentication scheme ( oauth | basic | bearer | apikey ) precedes to credential
-    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey)\s)?"
+    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey|ssws|ntlm)\s)?"
     value = r"(?P<value>" \
             r"(?(value_leftquote)" \
             r"(" \

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/deep_scanner/bzip2_scanner.py

@@ -29,7 +29,7 @@ class Bzip2Scanner(AbstractScanner, ABC):
             bzip2_content_provider = DataContentProvider(data=bz2.decompress(data_provider.data),
                                                          file_path=new_path,
                                                          file_type=Util.get_extension(new_path),
-                                                         info=f"{data_provider.info}|BZIP2:{new_path}")
+                                                         info=f"{data_provider.info}|BZIP2:{file_path}")
             new_limit = recursive_limit_size - len(bzip2_content_provider.data)
             bzip2_candidates = self.recursive_scan(bzip2_content_provider, depth, new_limit)
             return bzip2_candidates

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/deep_scanner/deep_scanner.py

@@ -76,17 +76,32 @@ class DeepScanner(
         return self.__scanner
 
     @staticmethod
-    def get_deep_scanners(data: bytes, file_type: str, depth: int) -> List[Any]:
-        """Returns possibly scan methods for the data depends on content"""
+    def get_deep_scanners(data: bytes, file_type: str, depth: int) -> Tuple[List[Any], List[Any]]:
+        """Returns possibly scan methods for the data depends on content and fallback scanners"""
         deep_scanners: List[Any] = []
+        fallback_scanners: List[Any] = []
         if Util.is_zip(data):
             if 0 < depth:
                 deep_scanners.append(ZipScanner)
-            # probably, there might be a docx, xlxs and so on.
+            # probably, there might be a docx, xlsx and so on.
             # It might be scanned with text representation in third-party libraries.
-            deep_scanners.append(XlsxScanner)
-            deep_scanners.append(DocxScanner)
-            deep_scanners.append(PptxScanner)
+            if file_type in (".xlsx", ".ods"):
+                deep_scanners.append(XlsxScanner)
+            else:
+                fallback_scanners.append(XlsxScanner)
+            if ".docx" == file_type:
+                deep_scanners.append(DocxScanner)
+            else:
+                fallback_scanners.append(DocxScanner)
+            if ".pptx" == file_type:
+                deep_scanners.append(PptxScanner)
+            else:
+                fallback_scanners.append(PptxScanner)
+        elif Util.is_com(data):
+            if ".xls" == file_type:
+                deep_scanners.append(XlsxScanner)
+            else:
+                fallback_scanners.append(XlsxScanner)
         elif Util.is_bzip2(data):
             if 0 < depth:
                 deep_scanners.append(Bzip2Scanner)
@@ -102,25 +117,67 @@ class DeepScanner(
             deep_scanners.append(JksScanner)
         elif Util.is_asn1(data):
             deep_scanners.append(Pkcs12Scanner)
-        elif file_type in [".eml", ".mht"]:
-            if Util.is_eml(data):
-                deep_scanners.append(EmlScanner)
-            elif Util.is_xml(data) and Util.is_html(data):
-                deep_scanners.append(HtmlScanner)
-            else:
-                deep_scanners.append(ByteScanner)
         elif Util.is_xml(data):
             if Util.is_html(data):
                 deep_scanners.append(HtmlScanner)
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
             elif Util.is_mxfile(data):
                 deep_scanners.append(MxfileScanner)
-            deep_scanners.append(XmlScanner)
-        else:
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
+            else:
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
+        elif Util.is_eml(data):
+            if ".eml" == file_type:
+                deep_scanners.append(EmlScanner)
+            else:
+                fallback_scanners.append(EmlScanner)
+            fallback_scanners.append(ByteScanner)
+        elif not Util.is_binary(data):
             if 0 < depth:
                 deep_scanners.append(EncoderScanner)
                 deep_scanners.append(LangScanner)
             deep_scanners.append(ByteScanner)
-        return deep_scanners
+        else:
+            logger.warning("Cannot apply a deep scanner for type %s", file_type)
+        return deep_scanners, fallback_scanners
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    def deep_scan_with_fallback(self, data_provider: DataContentProvider, depth: int,
+                                recursive_limit_size: int) -> List[Candidate]:
+        """Scans with deep scanners and fallback scanners if possible
+
+            Args:
+                data_provider: DataContentProvider with raw data
+                depth: maximal level of recursion
+                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
+
+            Returns: list with candidates
+
+        """
+        candidates: List[Candidate] = []
+        deep_scanners, fallback_scanners = self.get_deep_scanners(data_provider.data, data_provider.file_type, depth)
+        fallback = True
+        for scan_class in deep_scanners:
+            new_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+            if new_candidates is None:
+                # scanner did not recognise the content type
+                continue
+            augment_candidates(candidates, new_candidates)
+            # this scan is successful, so fallback is not necessary
+            fallback = False
+        if fallback:
+            for scan_class in deep_scanners:
+                fallback_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+                if fallback_candidates is None:
+                    continue
+                augment_candidates(candidates, fallback_candidates)
+                # use only first successful fallback scanner
+                break
+        return candidates
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
@@ -160,17 +217,8 @@ class DeepScanner(
                                                 file_path=content_provider.file_path,
                                                 file_type=content_provider.file_type,
                                                 info=content_provider.info or info)
-            # iterate for all possibly scanner methods WITHOUT ByteContentProvider for TextContentProvider
-            scanner_classes = self.get_deep_scanners(data, content_provider.file_type, depth)
-            fallback = True
-            for scan_class in scanner_classes:
-                if new_candidates := scan_class.data_scan(self, data_provider, depth, recursive_limit_size - len(data)):
-                    augment_candidates(candidates, new_candidates)
-                    fallback = False
-            if fallback and ByteScanner not in scanner_classes and not Util.is_binary(data):
-                # wrong assumption case
-                fallback_candidates = ByteScanner.data_scan(self, data_provider, depth, recursive_limit_size)
-                augment_candidates(candidates, fallback_candidates)
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size - len(data))
+            augment_candidates(candidates, new_candidates)
         return candidates
 
     def recursive_scan(
@@ -203,16 +251,8 @@ class DeepScanner(
                                                             FilePathExtractor.FIND_BY_EXT_RULE)
             candidates.append(dummy_candidate)
         else:
-            fallback = True
-            # iterate for all possibly scanner methods
-            scanner_classes = self.get_deep_scanners(data_provider.data, data_provider.file_type, depth)
-            for scanner_class in scanner_classes:
-                if new_candidates := scanner_class.data_scan(self, data_provider, depth, recursive_limit_size):
-                    augment_candidates(candidates, new_candidates)
-                    fallback = False
-            if fallback and ByteScanner not in scanner_classes and not Util.is_binary(data_provider.data):
-                bypass_candidates = ByteScanner.data_scan(self, data_provider, depth, recursive_limit_size)
-                augment_candidates(candidates, bypass_candidates)
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size)
+            augment_candidates(candidates, new_candidates)
 
         return candidates
 

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/deep_scanner/gzip_scanner.py

@@ -31,7 +31,7 @@ class GzipScanner(AbstractScanner, ABC):
                 gzip_content_provider = DataContentProvider(data=f.read(),
                                                             file_path=new_path,
                                                             file_type=Util.get_extension(new_path),
-                                                            info=f"{data_provider.info}|GZIP:{new_path}")
+                                                            info=f"{data_provider.info}|GZIP:{file_path}")
                 new_limit = recursive_limit_size - len(gzip_content_provider.data)
                 gzip_candidates = self.recursive_scan(gzip_content_provider, depth, new_limit)
                 return gzip_candidates

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/ml_model/features/word_in_path.py

@@ -21,8 +21,10 @@ class WordInPath(WordIn):
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         # actually there must be one path because the candidates are grouped before
-        if path := candidates[0].line_data_list[0].path:
-            posix_lower_path = Path(path).as_posix().lower()
+        if file_path := candidates[0].line_data_list[0].path:
+            path = Path(file_path)
+            # apply ./ for normalised path to detect "/src" for relative path
+            posix_lower_path = path.as_posix().lower() if path.is_absolute() else f"./{path.as_posix().lower()}"
             return self.word_in_str(posix_lower_path)
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/rules/config.yaml

@@ -576,7 +576,7 @@
   confidence: strong
   type: pattern
   values:
-    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>xox[aboprst]\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>xox[a-z]\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
   filter_type: GeneralPattern
   required_substrings:
     - xox
@@ -1407,6 +1407,37 @@
     - code
     - doc
 
+- name: Sentry Organization Auth Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntrys_eyJ[0-9A-Za-z_-]{80,8000}=*([0-9A-Za-z_-]{32,256})?)(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+    - ValueEntropyBase64Check
+  required_substrings:
+    - sntrys_eyJ
+  target:
+    - code
+    - doc
+
+- name: Sentry User Auth Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntryu_[0-9a-f]{64})(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+  required_substrings:
+    - sntryu_
+  target:
+    - code
+    - doc
+
 - name: Discord Bot Token
   severity: high
   confidence: strong

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/scanner/scanner.py

@@ -69,6 +69,7 @@ class Scanner:
             rule_path = APP_PATH / "rules" / "config.yaml"
         rule_templates = Util.yaml_load(rule_path)
         if rule_templates and isinstance(rule_templates, list):
+            rule_names = set()
             for rule_template in rule_templates:
                 try:
                     rule = Rule(self.config, rule_template)
@@ -77,6 +78,10 @@ class Scanner:
                     raise exc
                 if not self._is_available(rule):
                     continue
+                if rule.rule_name in rule_names:
+                    raise RuntimeError(f"Duplicated rule name {rule.rule_name}")
+                else:
+                    rule_names.add(rule.rule_name)
                 if 0 < rule.min_line_len:
                     if rule.rule_type == RuleType.KEYWORD:
                         self.min_keyword_len = min(self.min_keyword_len, rule.min_line_len)
@@ -141,7 +146,7 @@ class Scanner:
             # "cache" - YAPF and pycharm formatters ...
             matched_keyword = \
                 target_line_stripped_len >= self.min_keyword_len and (  #
-                    '=' in target_line_stripped or ':' in target_line_stripped)  #
+                        '=' in target_line_stripped or ':' in target_line_stripped)  #
             matched_pem_key = \
                 target_line_stripped_len >= self.min_pem_key_len \
                 and PEM_BEGIN_PATTERN in target_line_stripped and "PRIVATE" in target_line_stripped

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/secret/config.json

@@ -12,6 +12,10 @@
             ".xlsx",
             ".docx",
             ".pptx",
+            ".xls",
+            ".odp",
+            ".ods",
+            ".odt",
             ".pdf"
         ],
         "extension": [
@@ -67,7 +71,6 @@
             ".webm",
             ".webp",
             ".woff",
-            ".xls",
             ".yuv"
         ],
         "path": [

{credsweeper-1.10.7 → credsweeper-1.10.8}/credsweeper/utils/util.py

@@ -153,19 +153,26 @@ class Util:
         return entropy < min_entropy
 
     @staticmethod
-    def is_binary(data: bytes) -> bool:
+    def is_known(data: bytes) -> bool:
         """
         Returns true if any recognized binary format found
-        or two zeroes sequence is found which never exists in text format (UTF-8, UTF-16)
-        UTF-32 is not supported
         """
         if Util.is_zip(data) \
                 or Util.is_gzip(data) \
                 or Util.is_tar(data) \
                 or Util.is_bzip2(data) \
+                or Util.is_com(data) \
                 or Util.is_pdf(data) \
                 or Util.is_elf(data):
             return True
+        return False
+
+    @staticmethod
+    def is_binary(data: bytes) -> bool:
+        """
+        Returns True when two zeroes sequence is found which never exists in text format (UTF-8, UTF-16)
+        UTF-32 is not supported
+        """
         if 0 <= data.find(b"\0\0", 0, MAX_LINE_LENGTH):
             return True
         non_ascii_cnt = 0
@@ -224,7 +231,7 @@ class Util:
             encodings = AVAILABLE_ENCODINGS
         for encoding in encodings:
             try:
-                if binary_suggest and LATIN_1 == encoding and Util.is_binary(content):
+                if binary_suggest and LATIN_1 == encoding and (Util.is_known(content) or Util.is_binary(content)):
                     # LATIN_1 may convert data (bytes in range 0x80:0xFF are transformed)
                     # so skip this encoding when checking binaries
                     logger.warning("Binary file detected")
@@ -390,6 +397,15 @@ class Util:
                     return False
         return False
 
+    @staticmethod
+    def is_com(data: bytes) -> bool:
+        """According https://en.wikipedia.org/wiki/List_of_file_signatures"""
+        if isinstance(data, bytes) and 8 < len(data):
+            if data.startswith(b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1"):
+                # Compound File Binary Format: doc, xls, ppt, msi, msg
+                return True
+        return False
+
     @staticmethod
     def is_tar(data: bytes) -> bool:
         """According https://en.wikipedia.org/wiki/List_of_file_signatures"""
@@ -520,10 +536,10 @@ class Util:
     def is_eml(data: Union[bytes, bytearray]) -> bool:
         """According to https://datatracker.ietf.org/doc/html/rfc822 lookup the fields: Date, From, To or Subject"""
         if isinstance(data, (bytes, bytearray)):
-            if ((b"\nDate:" in data or data.startswith(b"Date:"))  #
-                    and (b"\nFrom:" in data or data.startswith(b"From:"))  #
-                    and (b"\nTo:" in data or data.startswith(b"To:")  #
-                         or b"\nSubject:" in data or data.startswith(b"Subject:"))):
+            if (b"\nDate:" in data or data.startswith(b"Date:")) \
+                    and (b"\nFrom:" in data or data.startswith(b"From:")) \
+                    and (b"\nTo:" in data or data.startswith(b"To:")) \
+                    and (b"\nSubject:" in data or data.startswith(b"Subject:")):
                 return True
         return False
 

{credsweeper-1.10.7 → credsweeper-1.10.8}/pyproject.toml

@@ -14,7 +14,8 @@ dependencies = [
     "lxml",
     "numpy<2.0.0",
     "odfpy",
-    "onnxruntime",
+    "onnxruntime==1.19.2; platform_system == 'Windows'",  # Python 3.9 limitation
+    "onnxruntime; platform_system != 'Windows'",
     "openpyxl",
     "pandas",
     "pdfminer.six",
@@ -25,6 +26,7 @@ dependencies = [
     "python-pptx",
     "PyYAML",
     "whatthepatch",
+    "xlrd",
 ]
 requires-python = ">=3.9"
 readme = "README.md"