credsweeper 1.12.1__tar.gz → 1.13.0__tar.gz

{credsweeper-1.12.1 → credsweeper-1.13.0}/.gitignore

@@ -149,3 +149,5 @@ fuzz/corpus/*
 # experiment result and data
 /experiment/results/*
 /experiment/data/*
+*.out
+

{credsweeper-1.12.1 → credsweeper-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.12.1
+Version: 1.13.0
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues
@@ -10,13 +10,12 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Security
 Classifier: Topic :: Software Development :: Quality Assurance
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: base58
 Requires-Dist: beautifulsoup4>=4.11.0
 Requires-Dist: colorama
@@ -24,10 +23,10 @@ Requires-Dist: cryptography
 Requires-Dist: gitpython
 Requires-Dist: humanfriendly
 Requires-Dist: lxml
-Requires-Dist: numpy<2.0.0
+Requires-Dist: numpy
 Requires-Dist: odfpy
-Requires-Dist: onnxruntime; platform_system != 'Windows'
-Requires-Dist: onnxruntime==1.19.2; platform_system == 'Windows'
+Requires-Dist: onnxruntime; platform_system != 'Windows' or python_version != '3.12'
+Requires-Dist: onnxruntime==1.19.2; platform_system == 'Windows' and python_version == '3.12'
 Requires-Dist: openpyxl
 Requires-Dist: pandas
 Requires-Dist: pdfminer-six
@@ -38,6 +37,7 @@ Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Requires-Dist: pyyaml
 Requires-Dist: rpmfile
+Requires-Dist: striprtf
 Requires-Dist: whatthepatch
 Requires-Dist: xlrd
 Description-Content-Type: text/markdown
@@ -90,7 +90,7 @@ Full documentation can be found here: <https://credsweeper.readthedocs.io/>
 
 ### Main Requirements
 
-- Python 3.9, 3.10, 3.11, 3.12
+- Python 3.10, 3.11, 3.12
 
 ### Installation
 

{credsweeper-1.12.1 → credsweeper-1.13.0}/README.md

@@ -46,7 +46,7 @@ Full documentation can be found here: <https://credsweeper.readthedocs.io/>
 
 ### Main Requirements
 
-- Python 3.9, 3.10, 3.11, 3.12
+- Python 3.10, 3.11, 3.12
 
 ### Installation
 

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/__init__.py

@@ -24,4 +24,4 @@ __all__ = [
     "__version__"
 ]
 
-__version__ = "1.12.1"
+__version__ = "1.13.0"

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/__main__.py

@@ -1,4 +1,5 @@
 import binascii
+import contextlib
 import logging
 import os
 import sys
@@ -34,24 +35,24 @@ def positive_int(value: Any) -> int:
     return int_value
 
 
-def threshold_or_float(arg: str) -> Union[float, ThresholdPreset]:
+def threshold_or_float_or_zero(arg: str) -> Union[int, float, ThresholdPreset]:
     """Return ThresholdPreset or a float from the input string
 
     Args:
         arg: string that either a float or one of allowed values in ThresholdPreset
 
     Returns:
-        float if arg convertible to float, ThresholdPreset if one of the allowed values
+        int = 0 to disable ML validator, float if arg convertible to float, ThresholdPreset if one of the allowed values
 
     Raises:
         ArgumentTypeError: if arg cannot be interpreted as float or ThresholdPreset
 
     """
     allowed_presents = [e.value for e in ThresholdPreset]
-    try:
+    if '0' == arg:
+        return 0
+    with contextlib.suppress(ValueError):
         return float(arg)  # try convert to float
-    except ValueError:
-        pass
     if arg in allowed_presents:
         return ThresholdPreset[arg]
     raise ArgumentTypeError(f"value must be a float or one of {allowed_presents}")
@@ -158,6 +159,10 @@ def get_arguments() -> Namespace:
                         help="find files by predefined extension",
                         dest="find_by_ext",
                         action="store_true")
+    parser.add_argument("--pedantic",
+                        help="process files without extension",
+                        action=BooleanOptionalAction,
+                        default=False)
     parser.add_argument("--depth",
                         help="additional recursive search in data (experimental)",
                         type=positive_int,
@@ -172,11 +177,11 @@ def get_arguments() -> Namespace:
                         "The lower the threshold - the more credentials will be reported. "
                         f"Allowed values: float between 0 and 1, or any of {[e.value for e in ThresholdPreset]} "
                         "(default: medium)",
-                        type=threshold_or_float,
+                        type=threshold_or_float_or_zero,
                         default=ThresholdPreset.medium,
                         dest="ml_threshold",
                         required=False,
-                        metavar="FLOAT_OR_STR")
+                        metavar="THRESHOLD_OR_FLOAT_OR_ZERO")
     parser.add_argument("--ml_batch_size",
                         "-b",
                         help="batch size for model inference (default: 16)",
@@ -299,6 +304,7 @@ def get_credsweeper(args: Namespace) -> CredSweeper:
                        ml_model=args.ml_model,
                        ml_providers=args.ml_providers,
                        find_by_ext=args.find_by_ext,
+                       pedantic=args.pedantic,
                        depth=args.depth,
                        doc=args.doc,
                        severity=args.severity,
@@ -335,7 +341,8 @@ def scan(args: Namespace, content_provider: AbstractProvider) -> int:
 def get_commit_providers(commit: Commit, repo: Repo) -> Sequence[ByteContentProvider]:
     """Process a commit and for providers"""
     result = {}
-    ancestors = commit.parents or [repo.tree()]
+    # use the hardcoded sha1 until sha256 objects are not supported by GitPython
+    ancestors = commit.parents or [repo.tree("4b825dc642cb6eb9a060e54bf8d69288fbee4904")]
     for parent in ancestors:
         for diff in parent.diff(commit):
             # only result files
@@ -372,9 +379,11 @@ def drill(args: Namespace) -> Tuple[int, int]:
         # then - credsweeper
         credsweeper = get_credsweeper(args)
         # use flat iterations to avoid recursive limits
-        to_scan = list(commits_sha1)
+        to_scan = set(commits_sha1)
         # local speedup for already scanned commits - avoid file system interactive
         scanned = set()
+        # to avoid double-check
+        skipped = set()
         while to_scan:
             commit_sha1 = to_scan.pop()
             if commit_sha1 in scanned:
@@ -382,8 +391,8 @@ def drill(args: Namespace) -> Tuple[int, int]:
                 continue
             commit = repo.commit(commit_sha1)
             if commit.parents:
-                # add parents anyway
-                to_scan.extend(x.hexsha for x in commit.parents)
+                # add parents only when they were not skipped or scanned previously
+                to_scan.update(x.hexsha for x in commit.parents if x.hexsha not in skipped and x.hexsha not in scanned)
             # check whether the commit has been checked and the report is present
             skip_already_scanned = False
             if args.json_filename:
@@ -401,9 +410,10 @@ def drill(args: Namespace) -> Tuple[int, int]:
                 else:
                     credsweeper.xlsx_filename = xlsx_path
             if skip_already_scanned:
-                logger.info("Skip already scanned commit: %s", commit_sha1)
+                skipped.add(commit_sha1)
+                logger.info("Skip already scanned commit: %s %s", commit_sha1, commit.committed_datetime.isoformat())
                 continue
-            logger.info("Scan commit: %s", commit_sha1)
+            logger.info("Scan commit: %s %s", commit_sha1, commit.committed_datetime.isoformat())
             # prepare all files to scan in the commit with bytes->IO transformation to avoid a multiprocess issue
             if providers := get_commit_providers(commit, repo):
                 credsweeper.credential_manager.candidates.clear()

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/app.py

@@ -52,11 +52,12 @@ class CredSweeper:
                  use_filters: bool = True,
                  pool_count: int = 1,
                  ml_batch_size: Optional[int] = None,
-                 ml_threshold: Union[float, ThresholdPreset] = ThresholdPreset.medium,
+                 ml_threshold: Union[int, float, ThresholdPreset] = ThresholdPreset.medium,
                  ml_config: Union[None, str, Path] = None,
                  ml_model: Union[None, str, Path] = None,
                  ml_providers: Optional[str] = None,
                  find_by_ext: bool = False,
+                 pedantic: bool = False,
                  depth: int = 0,
                  doc: bool = False,
                  severity: Union[Severity, str] = Severity.INFO,
@@ -86,6 +87,7 @@ class CredSweeper:
             ml_model: str or Path to set custom ml model
             ml_providers: str - comma separated list with providers
             find_by_ext: boolean - files will be reported by extension
+            pedantic: boolean - scan all files
             depth: int - how deep container files will be scanned
             doc: boolean - document-specific scanning
             severity: Severity - minimum severity level of rule
@@ -103,6 +105,7 @@ class CredSweeper:
         config_dict = self._get_config_dict(config_path=config_path,
                                             use_filters=use_filters,
                                             find_by_ext=find_by_ext,
+                                            pedantic=pedantic,
                                             depth=depth,
                                             doc=doc,
                                             severity=_severity,
@@ -145,6 +148,7 @@ class CredSweeper:
             config_path: Optional[str],  #
             use_filters: bool,  #
             find_by_ext: bool,  #
+            pedantic: bool,  #
             depth: int,  #
             doc: bool,  #
             severity: Severity,  #
@@ -155,6 +159,7 @@ class CredSweeper:
         config_dict["use_filters"] = use_filters
         config_dict["find_by_ext"] = find_by_ext
         config_dict["size_limit"] = size_limit
+        config_dict["pedantic"] = pedantic
         config_dict["depth"] = depth
         config_dict["doc"] = doc
         config_dict["severity"] = severity.value
@@ -169,7 +174,7 @@ class CredSweeper:
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
     def _use_ml_validation(self) -> bool:
-        if isinstance(self.ml_threshold, (float, int)) and 0 >= self.ml_threshold:
+        if isinstance(self.ml_threshold, int) and 0 == self.ml_threshold:
             logger.info("ML validation is disabled")
             return False
         if not self.credential_manager.candidates:

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/common/keyword_pattern.py

@@ -3,7 +3,10 @@ import re
 
 class KeywordPattern:
     """Pattern set of keyword types"""
-    directive = r"(?P<directive>(?:(?:[#%]define|%global)(?:\s|\\t)|\bset))?"
+    directive = r"(?P<directive>(?:" \
+                r"(?:[#%]define|define(?=(\s|\\{1,8}[tnr])*\()|%global)" \
+                r"(?:\s?\(|\s|\\{1,8}[tnr]){1,8}|\bset(?=\b|\w*(\s|\\{1,8}[tnr])*\()" \
+                r"))?"
     key_left = r"(?:\\[nrt]|(\\\\*u00|%)[0-9a-f]{2}|\s)*" \
                r"(?P<variable>(([\"'`]{1,8}[^:=\"'`}<>\\/&?]*|[^:=\"'`}<>\s()\\/&?;,%]*)"
     # keyword will be inserted here
@@ -13,7 +16,7 @@ class KeywordPattern:
                 r")"  # <variable>
     separator = r"(?(directive)|(\s|\\{1,8}[tnr])*\]?(\s|\\{1,8}[tnr])*)" \
                 r"(?P<separator>:(\s[a-z]{3,9}[?]?\s)?=|:(?!:)|=(>|&gt;|(\\\\*u00|%)26gt;)|!==|!=|===|==|=~|=" \
-                r"|(?(directive)(\\t|\s|\((?!\))){1,80}|%3d))" \
+                r"|(?(directive)(,|\\t|\s|\((?!\))){1,80}|%3d))" \
                 r"(\s|\\{1,8}[tnr])*"
     # might be curly, square or parenthesis with words before
     wrap = r"(?P<wrap>(" \
@@ -23,7 +26,7 @@ class KeywordPattern:
            r"\s*" \
            r"(\[(?!\])|\((?!\))|\{(?!\}))" \
            r"(\s|\\{1,8}[tnr])*" \
-           r"(?(get)('[^']+'|\"[^\"]+\")\s*,\s*|)" \
+           r"(?(get)('[^']{1,31}'|\"[^\"]{1,31}\")\s*,\s*|)" \
            r"([0-9a-z_]{1,32}\s*[:=]\s*)?" \
            r"){1,8})?"
     string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[\"'`])))?"

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/common/morpheme_checklist.txt

@@ -14,11 +14,15 @@
 /var
 000
 111
+14159265
+18284590
 222
 333
 444
 555
+65358979
 666
+71828182
 777
 80211
 888
@@ -195,7 +199,7 @@ aux
 avail
 avatar
 aver
-awesome
+awesom
 axis
 azure
 back
@@ -227,6 +231,7 @@ bind
 bio
 bipol
 bit
+bixby
 black
 blan
 bless
@@ -497,6 +502,7 @@ dust
 dvb
 dynamic
 dynamo
+eadbee
 easin
 easy
 ecdhe
@@ -789,6 +795,7 @@ jpg_
 json
 jump
 justif
+kafka
 kerberos
 kernel
 key
@@ -798,6 +805,7 @@ kind
 kinesis
 kirk
 know
+knox
 kris
 lab
 lag
@@ -1317,6 +1325,7 @@ sock
 soft
 solid
 solve
+some
 sony
 sort
 sound
@@ -1429,6 +1438,7 @@ tio
 tish
 title
 titud
+tizen
 tmp/
 to_
 tod
@@ -1440,6 +1450,7 @@ topic
 tory
 total
 touch
+tour
 trace
 tract
 traffic
@@ -1574,5 +1585,6 @@ you
 zeppelin
 zero
 zing
+zigbee
 zona
 zorro

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/config/config.py

@@ -35,6 +35,7 @@ class Config:
         self.candidate_output: List[str] = config["candidate_output"]
         self.find_by_ext: bool = config["find_by_ext"]
         self.size_limit: Optional[int] = parse_size(config["size_limit"]) if config["size_limit"] is not None else None
+        self.pedantic: bool = bool(config["pedantic"])
         self.depth: int = int(config["depth"])
         self.doc: bool = config["doc"]
         self.severity: Severity = Severity.get(config.get("severity"))

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/credentials/line_data.py

@@ -163,6 +163,7 @@ class LineData:
             self.clean_url_parameters()
             self.clean_bash_parameters()
             self.clean_toml_parameters()
+            self.clean_tag_parameters()
             if 0 <= self.value_start and 0 <= self.value_end and len(self.value) < len(_value):
                 start = _value.find(self.value)
                 self.value_start += start
@@ -232,6 +233,21 @@ class LineData:
                     self.value = self.value[:-1]
                     cleaning_required = True
 
+    def clean_tag_parameters(self) -> None:
+        """Remove closing tag from value if the opened is somewhere before in line"""
+        cleaning_required = self.value and self.value.endswith('>')
+        while cleaning_required:
+            closing_tag_pos = self.value.rfind("</")
+            if 0 <= closing_tag_pos:
+                # use `<a` to avoid tag parameters
+                opening_tag_prefix = f"<{self.value[closing_tag_pos + 2:-1]}"
+                if cleaning_required := (opening_tag_prefix not in self.value
+                                         and 0 <= self.line.find(opening_tag_prefix, 0, self.value_start)):
+                    self.value = self.value[:closing_tag_pos]
+                    cleaning_required = self.value and self.value.endswith('>')
+            else:
+                break
+
     def sanitize_variable(self) -> None:
         """Remove trailing spaces, dashes and quotations around the variable. Correct position."""
         sanitized_var_len = 0

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/deep_scanner/deep_scanner.py

@@ -1,7 +1,6 @@
 import logging
 from typing import List, Any, Tuple
 
-from credsweeper.common.constants import MIN_DATA_LEN
 from credsweeper.config.config import Config
 from credsweeper.scanner.scanner import Scanner
 from credsweeper.utils.util import Util
@@ -23,7 +22,9 @@ from .pdf_scanner import PdfScanner
 from .pkcs_scanner import PkcsScanner
 from .pptx_scanner import PptxScanner
 from .rpm_scanner import RpmScanner
+from .rtf_scanner import RtfScanner
 from .sqlite3_scanner import Sqlite3Scanner
+from .strings_scanner import StringsScanner
 from .tar_scanner import TarScanner
 from .tmx_scanner import TmxScanner
 from .xlsx_scanner import XlsxScanner
@@ -49,8 +50,10 @@ class DeepScanner(
     PdfScanner,  #
     PkcsScanner,  #
     PptxScanner,  #
+    RtfScanner,  #
     RpmScanner,  #
     Sqlite3Scanner,  #
+    StringsScanner,  #
     TarScanner,  #
     DebScanner,  #
     XmlScanner,  #
@@ -133,6 +136,9 @@ class DeepScanner(
                 deep_scanners.append(Sqlite3Scanner)
         elif Util.is_asn1(data):
             deep_scanners.append(PkcsScanner)
+        elif Util.is_rtf(data):
+            deep_scanners.append(RtfScanner)
+            fallback_scanners.append(ByteScanner)
         elif Util.is_xml(data):
             if Util.is_html(data):
                 deep_scanners.append(HtmlScanner)
@@ -150,7 +156,7 @@ class DeepScanner(
                 deep_scanners.append(XmlScanner)
                 fallback_scanners.append(ByteScanner)
         elif Util.is_eml(data):
-            if ".eml" == descriptor.extension:
+            if descriptor.extension in (".eml", ".mht"):
                 deep_scanners.append(EmlScanner)
             else:
                 if 0 < depth:
@@ -158,9 +164,6 @@ class DeepScanner(
                     deep_scanners.append(PatchScanner)
                 fallback_scanners.append(EmlScanner)
             fallback_scanners.append(ByteScanner)
-        elif Util.is_known(data):
-            # the format is known but cannot be scanned
-            pass
         elif not Util.is_binary(data):
             if 0 < depth:
                 deep_scanners.append(PatchScanner)
@@ -168,6 +171,9 @@ class DeepScanner(
                 deep_scanners.append(LangScanner)
             deep_scanners.append(ByteScanner)
         else:
-            logger.warning("Cannot apply a deep scanner for type %s prefix %s %d", descriptor,
-                           repr(data[:MIN_DATA_LEN]), len(data))
+            if 0 < depth:
+                deep_scanners.append(StringsScanner)
+            else:
+                logger.warning("Cannot apply a deep scanner for type %s prefix %s %d", descriptor, repr(data[:32]),
+                               len(data))
         return deep_scanners, fallback_scanners

credsweeper-1.13.0/credsweeper/deep_scanner/rtf_scanner.py

@@ -0,0 +1,41 @@
+import logging
+from abc import ABC
+from typing import List, Optional
+
+from striprtf import striprtf
+
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.string_content_provider import StringContentProvider
+from credsweeper.utils.util import Util
+
+logger = logging.getLogger(__name__)
+
+
+class RtfScanner(AbstractScanner, ABC):
+    """Implements squash file system scanning"""
+
+    @staticmethod
+    def get_lines(text: str) -> List[str]:
+        """Extracts text lines from RTF format"""
+        rtf_text = striprtf.rtf_to_text(text)
+        lines = Util.split_text(rtf_text)
+        return lines
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Scans data as RTF"""
+        try:
+            string_data_provider = StringContentProvider(lines=RtfScanner.get_lines(data_provider.text),
+                                                         file_path=data_provider.file_path,
+                                                         file_type=data_provider.file_type,
+                                                         info=f"{data_provider.info}|RTF")
+            rtf_candidates = self.scanner.scan(string_data_provider)
+            return rtf_candidates
+        except Exception as rtf_exc:
+            logger.error(f"{data_provider.file_path}:{rtf_exc}")
+        return None

credsweeper-1.13.0/credsweeper/deep_scanner/strings_scanner.py

@@ -0,0 +1,52 @@
+import logging
+from abc import ABC
+from typing import List, Optional, Tuple
+
+from credsweeper.common.constants import MIN_DATA_LEN
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.string_content_provider import StringContentProvider
+
+logger = logging.getLogger(__name__)
+
+
+class StringsScanner(AbstractScanner, ABC):
+    """Implements known binary file scanning with ASCII strings representations"""
+
+    @staticmethod
+    def get_strings(data: bytes) -> List[Tuple[str, int]]:
+        """Processes binary to found ASCII strings. Use offset instead line number."""
+        strings = []
+        offset = 0
+        line = ''
+        for n, x in enumerate(data):
+            if 0x09 == x or 0x20 <= x <= 0x7E:
+                # TAB, SPACE and visible ASCII symbols
+                if not offset:
+                    # for line number
+                    offset = n
+                line += chr(x)
+            elif MIN_DATA_LEN <= len(line):
+                strings.append((line, offset))
+                offset = 0
+                line = ''
+        if MIN_DATA_LEN <= len(line):
+            strings.append((line, offset))
+        return strings
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Extracts data file from .ar (debian) archive and launches data_scan"""
+
+        if strings := StringsScanner.get_strings(data_provider.data):
+            string_data_provider = StringContentProvider(lines=[x[0] for x in strings],
+                                                         line_numbers=[x[1] for x in strings],
+                                                         file_path=data_provider.file_path,
+                                                         file_type=data_provider.file_type,
+                                                         info=f"{data_provider.info}|STRINGS")
+            return self.scanner.scan(string_data_provider)
+        return None if strings is None else []

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/file_handler/byte_content_provider.py

@@ -1,3 +1,4 @@
+import logging
 from functools import cached_property
 from typing import List, Optional, Generator
 
@@ -5,6 +6,8 @@ from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.file_handler.content_provider import ContentProvider
 from credsweeper.utils.util import Util
 
+logger = logging.getLogger(__name__)
+
 
 class ByteContentProvider(ContentProvider):
     """Allow to scan byte sequence instead of extra reading a file"""
@@ -42,7 +45,13 @@ class ByteContentProvider(ContentProvider):
     def lines(self) -> List[str]:
         """lines RO getter for ByteContentProvider"""
         if self.__lines is None:
-            self.__lines = Util.decode_bytes(self.__data)
+            text = Util.decode_text(self.__data)
+            if text is None:
+                logger.warning("Binary data detected %s %s %s", self.file_path, self.info,
+                               repr(self.__data[:32]) if isinstance(self.__data, bytes) else "NONE")
+                self.__lines = []
+            else:
+                self.__lines = Util.split_text(text)
         return self.__lines if self.__lines is not None else []
 
     def yield_analysis_target(self, min_len: int) -> Generator[AnalysisTarget, None, None]:

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/file_handler/file_path_extractor.py

@@ -127,6 +127,8 @@ class FilePathExtractor:
         Return:
             True when the file full path should be excluded according config
         """
+        if config.pedantic:
+            return False
         path = path.replace('\\', '/')
         lower_path = path.lower()
         if config.not_allowed_path_pattern.match(lower_path):

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/file_handler/text_content_provider.py

@@ -54,7 +54,13 @@ class TextContentProvider(ContentProvider):
     def lines(self) -> Optional[List[str]]:
         """lines getter for TextContentProvider"""
         if self.__lines is None:
-            self.__lines = Util.decode_bytes(self.data)
+            text = Util.decode_text(self.data)
+            if text is None:
+                logger.warning("Binary file detected %s %s %s", self.file_path, self.info,
+                               repr(self.__data[:32]) if isinstance(self.__data, bytes) else "NONE")
+                self.__lines = []
+            else:
+                self.__lines = Util.split_text(text)
         return self.__lines if self.__lines is not None else []
 
     def yield_analysis_target(self, min_len: int) -> Generator[AnalysisTarget, None, None]:

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/filters/__init__.py

@@ -13,7 +13,6 @@ from credsweeper.filters.value_base64_part_check import ValueBase64PartCheck
 from credsweeper.filters.value_basic_auth_check import ValueBasicAuthCheck
 from credsweeper.filters.value_blocklist_check import ValueBlocklistCheck
 from credsweeper.filters.value_camel_case_check import ValueCamelCaseCheck
-from credsweeper.filters.value_couple_keyword_check import ValueCoupleKeywordCheck
 from credsweeper.filters.value_dictionary_keyword_check import ValueDictionaryKeywordCheck
 from credsweeper.filters.value_discord_bot_check import ValueDiscordBotCheck
 from credsweeper.filters.value_entropy_base32_check import ValueEntropyBase32Check
@@ -30,6 +29,7 @@ from credsweeper.filters.value_json_web_token_check import ValueJsonWebTokenChec
 from credsweeper.filters.value_last_word_check import ValueLastWordCheck
 from credsweeper.filters.value_length_check import ValueLengthCheck
 from credsweeper.filters.value_method_check import ValueMethodCheck
+from credsweeper.filters.value_morphemes_check import ValueMorphemesCheck
 from credsweeper.filters.value_not_allowed_pattern_check import ValueNotAllowedPatternCheck
 from credsweeper.filters.value_not_part_encoded_check import ValueNotPartEncodedCheck
 from credsweeper.filters.value_number_check import ValueNumberCheck

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/filters/group/token_pattern.py

@@ -1,6 +1,6 @@
 from credsweeper.common.constants import GroupType
 from credsweeper.config.config import Config
-from credsweeper.filters import ValueCoupleKeywordCheck, ValueCamelCaseCheck, ValueNumberCheck, ValuePatternCheck
+from credsweeper.filters import ValueMorphemesCheck, ValueCamelCaseCheck, ValueNumberCheck, ValuePatternCheck
 from credsweeper.filters.group.group import Group
 
 
@@ -10,7 +10,7 @@ class TokenPattern(Group):
     def __init__(self, config: Config) -> None:
         super().__init__(config, GroupType.DEFAULT)
         self.filters = [
-            ValueCoupleKeywordCheck(),
+            ValueMorphemesCheck(),
             ValueNumberCheck(),
             ValueCamelCaseCheck(),
             ValuePatternCheck(),

{credsweeper-1.12.1 → credsweeper-1.13.0}/credsweeper/filters/group/weird_base36_token.py

@@ -1,6 +1,6 @@
 from credsweeper.common.constants import GroupType
 from credsweeper.config.config import Config
-from credsweeper.filters import ValueCoupleKeywordCheck, ValuePatternCheck, ValueNumberCheck, ValueEntropyBase36Check, \
+from credsweeper.filters import ValueMorphemesCheck, ValuePatternCheck, ValueNumberCheck, ValueEntropyBase36Check, \
     ValueTokenBase36Check
 from credsweeper.filters.group.group import Group
 
@@ -11,7 +11,7 @@ class WeirdBase36Token(Group):
     def __init__(self, config: Config) -> None:
         super().__init__(config, GroupType.DEFAULT)
         self.filters = [
-            ValueCoupleKeywordCheck(),
+            ValueMorphemesCheck(threshold=1),
             ValuePatternCheck(),
             ValueNumberCheck(),
             ValueTokenBase36Check(),