credsweeper 1.10.7__tar.gz → 1.11.0__tar.gz

{credsweeper-1.10.7 → credsweeper-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.10.7
+Version: 1.11.0
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues
@@ -26,7 +26,8 @@ Requires-Dist: humanfriendly
 Requires-Dist: lxml
 Requires-Dist: numpy<2.0.0
 Requires-Dist: odfpy
-Requires-Dist: onnxruntime
+Requires-Dist: onnxruntime; platform_system != 'Windows'
+Requires-Dist: onnxruntime==1.19.2; platform_system == 'Windows'
 Requires-Dist: openpyxl
 Requires-Dist: pandas
 Requires-Dist: pdfminer-six
@@ -37,6 +38,7 @@ Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Requires-Dist: pyyaml
 Requires-Dist: whatthepatch
+Requires-Dist: xlrd
 Description-Content-Type: text/markdown
 
 # CredSweeper

{credsweeper-1.10.7 → credsweeper-1.11.0}/credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.10.7"
+__version__ = "1.11.0"

{credsweeper-1.10.7 → credsweeper-1.11.0}/credsweeper/common/keyword_pattern.py

@@ -26,7 +26,7 @@ class KeywordPattern:
     string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[`'\"])))?"
     left_quote = r"(?P<value_leftquote>((?P<esq>\\{1,8})?([`'\"]|&(quot|apos);)){1,4}))?"
     # Authentication scheme ( oauth | basic | bearer | apikey ) precedes to credential
-    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey)\s)?"
+    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey|ssws|ntlm)\s)?"
     value = r"(?P<value>" \
             r"(?(value_leftquote)" \
             r"(" \

{credsweeper-1.10.7 → credsweeper-1.11.0}/credsweeper/common/morpheme_checklist.txt

@@ -960,6 +960,7 @@ nish
 nism
 node
 non
+nope
 norm
 not
 nsive
@@ -1529,6 +1530,7 @@ warn
 watch
 wave
 way
+weak
 web
 week
 weight

{credsweeper-1.10.7 → credsweeper-1.11.0}/credsweeper/deep_scanner/bzip2_scanner.py

@@ -29,7 +29,7 @@ class Bzip2Scanner(AbstractScanner, ABC):
             bzip2_content_provider = DataContentProvider(data=bz2.decompress(data_provider.data),
                                                          file_path=new_path,
                                                          file_type=Util.get_extension(new_path),
-                                                         info=f"{data_provider.info}|BZIP2:{new_path}")
+                                                         info=f"{data_provider.info}|BZIP2:{file_path}")
             new_limit = recursive_limit_size - len(bzip2_content_provider.data)
             bzip2_candidates = self.recursive_scan(bzip2_content_provider, depth, new_limit)
             return bzip2_candidates

{credsweeper-1.10.7 → credsweeper-1.11.0}/credsweeper/deep_scanner/deep_scanner.py

@@ -76,17 +76,32 @@ class DeepScanner(
         return self.__scanner
 
     @staticmethod
-    def get_deep_scanners(data: bytes, file_type: str, depth: int) -> List[Any]:
-        """Returns possibly scan methods for the data depends on content"""
+    def get_deep_scanners(data: bytes, file_type: str, depth: int) -> Tuple[List[Any], List[Any]]:
+        """Returns possibly scan methods for the data depends on content and fallback scanners"""
         deep_scanners: List[Any] = []
+        fallback_scanners: List[Any] = []
         if Util.is_zip(data):
             if 0 < depth:
                 deep_scanners.append(ZipScanner)
-            # probably, there might be a docx, xlxs and so on.
+            # probably, there might be a docx, xlsx and so on.
             # It might be scanned with text representation in third-party libraries.
-            deep_scanners.append(XlsxScanner)
-            deep_scanners.append(DocxScanner)
-            deep_scanners.append(PptxScanner)
+            if file_type in (".xlsx", ".ods"):
+                deep_scanners.append(XlsxScanner)
+            else:
+                fallback_scanners.append(XlsxScanner)
+            if ".docx" == file_type:
+                deep_scanners.append(DocxScanner)
+            else:
+                fallback_scanners.append(DocxScanner)
+            if ".pptx" == file_type:
+                deep_scanners.append(PptxScanner)
+            else:
+                fallback_scanners.append(PptxScanner)
+        elif Util.is_com(data):
+            if ".xls" == file_type:
+                deep_scanners.append(XlsxScanner)
+            else:
+                fallback_scanners.append(XlsxScanner)
         elif Util.is_bzip2(data):
             if 0 < depth:
                 deep_scanners.append(Bzip2Scanner)
@@ -102,25 +117,67 @@ class DeepScanner(
             deep_scanners.append(JksScanner)
         elif Util.is_asn1(data):
             deep_scanners.append(Pkcs12Scanner)
-        elif file_type in [".eml", ".mht"]:
-            if Util.is_eml(data):
-                deep_scanners.append(EmlScanner)
-            elif Util.is_xml(data) and Util.is_html(data):
-                deep_scanners.append(HtmlScanner)
-            else:
-                deep_scanners.append(ByteScanner)
         elif Util.is_xml(data):
             if Util.is_html(data):
                 deep_scanners.append(HtmlScanner)
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
             elif Util.is_mxfile(data):
                 deep_scanners.append(MxfileScanner)
-            deep_scanners.append(XmlScanner)
-        else:
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
+            else:
+                deep_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
+        elif Util.is_eml(data):
+            if ".eml" == file_type:
+                deep_scanners.append(EmlScanner)
+            else:
+                fallback_scanners.append(EmlScanner)
+            fallback_scanners.append(ByteScanner)
+        elif not Util.is_binary(data):
             if 0 < depth:
                 deep_scanners.append(EncoderScanner)
                 deep_scanners.append(LangScanner)
             deep_scanners.append(ByteScanner)
-        return deep_scanners
+        else:
+            logger.warning("Cannot apply a deep scanner for type %s", file_type)
+        return deep_scanners, fallback_scanners
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    def deep_scan_with_fallback(self, data_provider: DataContentProvider, depth: int,
+                                recursive_limit_size: int) -> List[Candidate]:
+        """Scans with deep scanners and fallback scanners if possible
+
+            Args:
+                data_provider: DataContentProvider with raw data
+                depth: maximal level of recursion
+                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
+
+            Returns: list with candidates
+
+        """
+        candidates: List[Candidate] = []
+        deep_scanners, fallback_scanners = self.get_deep_scanners(data_provider.data, data_provider.file_type, depth)
+        fallback = True
+        for scan_class in deep_scanners:
+            new_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+            if new_candidates is None:
+                # scanner did not recognise the content type
+                continue
+            augment_candidates(candidates, new_candidates)
+            # this scan is successful, so fallback is not necessary
+            fallback = False
+        if fallback:
+            for scan_class in deep_scanners:
+                fallback_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+                if fallback_candidates is None:
+                    continue
+                augment_candidates(candidates, fallback_candidates)
+                # use only first successful fallback scanner
+                break
+        return candidates
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
@@ -160,17 +217,8 @@ class DeepScanner(
                                                 file_path=content_provider.file_path,
                                                 file_type=content_provider.file_type,
                                                 info=content_provider.info or info)
-            # iterate for all possibly scanner methods WITHOUT ByteContentProvider for TextContentProvider
-            scanner_classes = self.get_deep_scanners(data, content_provider.file_type, depth)
-            fallback = True
-            for scan_class in scanner_classes:
-                if new_candidates := scan_class.data_scan(self, data_provider, depth, recursive_limit_size - len(data)):
-                    augment_candidates(candidates, new_candidates)
-                    fallback = False
-            if fallback and ByteScanner not in scanner_classes and not Util.is_binary(data):
-                # wrong assumption case
-                fallback_candidates = ByteScanner.data_scan(self, data_provider, depth, recursive_limit_size)
-                augment_candidates(candidates, fallback_candidates)
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size - len(data))
+            augment_candidates(candidates, new_candidates)
         return candidates
 
     def recursive_scan(
@@ -203,16 +251,8 @@ class DeepScanner(
                                                             FilePathExtractor.FIND_BY_EXT_RULE)
             candidates.append(dummy_candidate)
         else:
-            fallback = True
-            # iterate for all possibly scanner methods
-            scanner_classes = self.get_deep_scanners(data_provider.data, data_provider.file_type, depth)
-            for scanner_class in scanner_classes:
-                if new_candidates := scanner_class.data_scan(self, data_provider, depth, recursive_limit_size):
-                    augment_candidates(candidates, new_candidates)
-                    fallback = False
-            if fallback and ByteScanner not in scanner_classes and not Util.is_binary(data_provider.data):
-                bypass_candidates = ByteScanner.data_scan(self, data_provider, depth, recursive_limit_size)
-                augment_candidates(candidates, bypass_candidates)
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size)
+            augment_candidates(candidates, new_candidates)
 
         return candidates
 

{credsweeper-1.10.7 → credsweeper-1.11.0}/credsweeper/deep_scanner/gzip_scanner.py

@@ -31,7 +31,7 @@ class GzipScanner(AbstractScanner, ABC):
                 gzip_content_provider = DataContentProvider(data=f.read(),
                                                             file_path=new_path,
                                                             file_type=Util.get_extension(new_path),
-                                                            info=f"{data_provider.info}|GZIP:{new_path}")
+                                                            info=f"{data_provider.info}|GZIP:{file_path}")
                 new_limit = recursive_limit_size - len(gzip_content_provider.data)
                 gzip_candidates = self.recursive_scan(gzip_content_provider, depth, new_limit)
                 return gzip_candidates

{credsweeper-1.10.7 → credsweeper-1.11.0}/credsweeper/ml_model/features/__init__.py

@@ -6,7 +6,9 @@ from credsweeper.ml_model.features.length_of_attribute import LengthOfAttribute
 from credsweeper.ml_model.features.morpheme_dense import MorphemeDense
 from credsweeper.ml_model.features.rule_name import RuleName
 from credsweeper.ml_model.features.search_in_attribute import SearchInAttribute
-from credsweeper.ml_model.features.word_in_line import WordInLine
 from credsweeper.ml_model.features.word_in_path import WordInPath
+from credsweeper.ml_model.features.word_in_postamble import WordInPostamble
+from credsweeper.ml_model.features.word_in_preamble import WordInPreamble
+from credsweeper.ml_model.features.word_in_transition import WordInTransition
 from credsweeper.ml_model.features.word_in_value import WordInValue
 from credsweeper.ml_model.features.word_in_variable import WordInVariable

{credsweeper-1.10.7 → credsweeper-1.11.0}/credsweeper/ml_model/features/word_in_path.py

@@ -21,8 +21,10 @@ class WordInPath(WordIn):
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
         # actually there must be one path because the candidates are grouped before
-        if path := candidates[0].line_data_list[0].path:
-            posix_lower_path = Path(path).as_posix().lower()
+        if file_path := candidates[0].line_data_list[0].path:
+            path = Path(file_path)
+            # apply ./ for normalised path to detect "/src" for relative path
+            posix_lower_path = path.as_posix().lower() if path.is_absolute() else f"./{path.as_posix().lower()}"
             return self.word_in_str(posix_lower_path)
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper-1.11.0/credsweeper/ml_model/features/word_in_postamble.py

@@ -0,0 +1,32 @@
+from typing import List
+
+import numpy as np
+
+from credsweeper.common.constants import ML_HUNK
+from credsweeper.credentials import Candidate
+from credsweeper.ml_model.features.word_in import WordIn
+
+
+class WordInPostamble(WordIn):
+    """Feature is true if line contains at least one word from predefined list."""
+
+    def __init__(self, words: List[str]) -> None:
+        """Feature returns array of matching words
+
+        Args:
+            words: list of predefined words - MUST BE IN LOWER CASE
+
+        """
+        super().__init__(words)
+
+    def extract(self, candidate: Candidate) -> np.ndarray:
+        """Returns true if any words in a part of line after value"""
+        postamble_end = len(candidate.line_data_list[0].line) \
+            if len(candidate.line_data_list[0].line) < candidate.line_data_list[0].value_end + ML_HUNK \
+            else candidate.line_data_list[0].value_end + ML_HUNK
+        postamble = candidate.line_data_list[0].line[candidate.line_data_list[0].value_end:postamble_end].strip()
+
+        if postamble:
+            return self.word_in_str(postamble.lower())
+        else:
+            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper-1.11.0/credsweeper/ml_model/features/word_in_preamble.py

@@ -0,0 +1,37 @@
+from typing import List
+
+import numpy as np
+
+from credsweeper.common.constants import ML_HUNK
+from credsweeper.credentials import Candidate
+from credsweeper.ml_model.features.word_in import WordIn
+
+
+class WordInPreamble(WordIn):
+    """Feature is true if line contains at least one word from predefined list."""
+
+    def __init__(self, words: List[str]) -> None:
+        """Feature returns array of matching words
+
+        Args:
+            words: list of predefined words - MUST BE IN LOWER CASE
+
+        """
+        super().__init__(words)
+
+    def extract(self, candidate: Candidate) -> np.ndarray:
+        """Returns true if any words in line before variable or value"""
+        if 0 <= candidate.line_data_list[0].variable_start:
+            preamble_start = 0 if ML_HUNK >= candidate.line_data_list[0].variable_start \
+                else candidate.line_data_list[0].variable_start - ML_HUNK
+            preamble = candidate.line_data_list[0].line[preamble_start:candidate.line_data_list[0].
+                                                        variable_start].strip()
+        else:
+            preamble_start = 0 if ML_HUNK >= candidate.line_data_list[0].value_start \
+                else candidate.line_data_list[0].value_start - ML_HUNK
+            preamble = candidate.line_data_list[0].line[preamble_start:candidate.line_data_list[0].value_start].strip()
+
+        if preamble:
+            return self.word_in_str(preamble.lower())
+        else:
+            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper-1.10.7/credsweeper/ml_model/features/word_in_line.py → credsweeper-1.11.0/credsweeper/ml_model/features/word_in_transition.py

@@ -2,13 +2,11 @@ from typing import List
 
 import numpy as np
 
-from credsweeper.common.constants import CHUNK_SIZE
 from credsweeper.credentials import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
-from credsweeper.utils import Util
 
 
-class WordInLine(WordIn):
+class WordInTransition(WordIn):
     """Feature is true if line contains at least one word from predefined list."""
 
     def __init__(self, words: List[str]) -> None:
@@ -21,9 +19,14 @@ class WordInLine(WordIn):
         super().__init__(words)
 
     def extract(self, candidate: Candidate) -> np.ndarray:
-        """Returns true if any words in first line"""
-        subtext = Util.subtext(candidate.line_data_list[0].line, candidate.line_data_list[0].value_start, CHUNK_SIZE)
-        if subtext:
-            return self.word_in_str(subtext.lower())
+        """Returns true if any words between variable and value"""
+        if 0 <= candidate.line_data_list[0].variable_end < candidate.line_data_list[0].value_start:
+            transition = candidate.line_data_list[0].line[candidate.line_data_list[0].variable_end:candidate.
+                                                          line_data_list[0].value_start].strip()
+        else:
+            transition = ''
+
+        if transition:
+            return self.word_in_str(transition.lower())
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])