credsweeper 1.11.5__py3-none-any.whl → 1.13.3__py3-none-any.whl

credsweeper/rules/rule.py

@@ -7,9 +7,10 @@ from typing import Dict, List, Optional, Union, Set
 from credsweeper import filters
 from credsweeper.common.constants import RuleType, Severity, MAX_LINE_LENGTH, Confidence
 from credsweeper.common.keyword_pattern import KeywordPattern
-from credsweeper.config import Config
-from credsweeper.filters import Filter, group
-from credsweeper.filters.group import Group
+from credsweeper.config.config import Config
+from credsweeper.filters import group
+from credsweeper.filters.filter import Filter
+from credsweeper.filters.group.group import Group
 
 logger = logging.getLogger(__name__)
 

credsweeper/scanner/__init__.py

@@ -1 +0,0 @@
-from credsweeper.scanner.scanner import Scanner

credsweeper/scanner/scan_type/__init__.py

@@ -1,5 +0,0 @@
-from credsweeper.scanner.scan_type.scan_type import ScanType  # isort:skip
-
-from credsweeper.scanner.scan_type.multi_pattern import MultiPattern
-from credsweeper.scanner.scan_type.pem_key_pattern import PemKeyPattern
-from credsweeper.scanner.scan_type.single_pattern import SinglePattern

credsweeper/scanner/scan_type/multi_pattern.py

@@ -1,11 +1,11 @@
 from typing import List
 
 from credsweeper.common.constants import RuleType
-from credsweeper.config import Config
-from credsweeper.credentials import Candidate
+from credsweeper.config.config import Config
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.rules import Rule
-from credsweeper.scanner.scan_type import ScanType
+from credsweeper.rules.rule import Rule
+from credsweeper.scanner.scan_type.scan_type import ScanType
 
 
 class MultiPattern(ScanType):

credsweeper/scanner/scan_type/pem_key_pattern.py

@@ -2,11 +2,11 @@ import logging
 from typing import List
 
 from credsweeper.common.constants import RuleType
-from credsweeper.config import Config
-from credsweeper.credentials import Candidate
+from credsweeper.config.config import Config
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.rules import Rule
-from credsweeper.scanner.scan_type import ScanType
+from credsweeper.rules.rule import Rule
+from credsweeper.scanner.scan_type.scan_type import ScanType
 from credsweeper.utils.pem_key_detector import PemKeyDetector
 
 logger = logging.getLogger(__name__)

credsweeper/scanner/scan_type/scan_type.py

@@ -4,11 +4,11 @@ from abc import ABC, abstractmethod
 from typing import List
 
 from credsweeper.common.constants import RuleType, MIN_DATA_LEN
-from credsweeper.config import Config
-from credsweeper.credentials import Candidate, LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.candidate import Candidate, LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.rules import Rule
+from credsweeper.filters.filter import Filter
+from credsweeper.rules.rule import Rule
 
 logger = logging.getLogger(__name__)
 

credsweeper/scanner/scan_type/single_pattern.py

@@ -1,10 +1,10 @@
 from typing import List
 
-from credsweeper.config import Config
-from credsweeper.credentials import Candidate
+from credsweeper.config.config import Config
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.rules import Rule
-from credsweeper.scanner.scan_type import ScanType
+from credsweeper.rules.rule import Rule
+from credsweeper.scanner.scan_type.scan_type import ScanType
 
 
 class SinglePattern(ScanType):

credsweeper/scanner/scanner.py

@@ -6,16 +6,21 @@ from typing import List, Type, Tuple, Union, Dict, Generator, Set
 from credsweeper.app import APP_PATH
 from credsweeper.common.constants import RuleType, MIN_VARIABLE_LENGTH, MIN_SEPARATOR_LENGTH, MIN_VALUE_LENGTH, \
     MAX_LINE_LENGTH, PEM_BEGIN_PATTERN
-from credsweeper.config import Config
-from credsweeper.credentials import Candidate
+from credsweeper.config.config import Config
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.file_handler.content_provider import ContentProvider
-from credsweeper.rules import Rule
-from credsweeper.scanner.scan_type import PemKeyPattern, ScanType, SinglePattern, MultiPattern
-from credsweeper.utils import Util
+from credsweeper.rules.rule import Rule
+from credsweeper.scanner.scan_type.multi_pattern import MultiPattern
+from credsweeper.scanner.scan_type.pem_key_pattern import PemKeyPattern
+from credsweeper.scanner.scan_type.scan_type import ScanType
+from credsweeper.scanner.scan_type.single_pattern import SinglePattern
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 
+RULES_PATH = APP_PATH / "rules" / "config.yaml"
+
 
 class Scanner:
     """Advanced Credential Scanner base class.
@@ -63,11 +68,11 @@ class Scanner:
                 return True
         return False
 
-    def _set_rules_scanners(self, rule_path: Union[None, str, Path]) -> None:
+    def _set_rules_scanners(self, rules_path: Union[None, str, Path]) -> None:
         """Auxiliary method to fill rules, determine min_pattern_len and set scanners"""
-        if rule_path is None:
-            rule_path = APP_PATH / "rules" / "config.yaml"
-        rule_templates = Util.yaml_load(rule_path)
+        if rules_path is None:
+            rules_path = RULES_PATH
+        rule_templates = Util.yaml_load(rules_path)
         if rule_templates and isinstance(rule_templates, list):
             rule_names = set()
             for rule_template in rule_templates:
@@ -95,7 +100,7 @@ class Scanner:
                         logger.warning(f"Unknown rule type:{rule.rule_type}")
                 self.rules_scanners.append((rule, self.get_scanner(rule)))
         else:
-            raise RuntimeError(f"Wrong rules '{rule_templates}' were read from '{rule_path}'")
+            raise RuntimeError(f"Wrong rules '{rule_templates}' were read from '{rules_path}'")
 
     def _is_available(self, rule: Rule) -> bool:
         """separate the method to reduce complexity"""
@@ -142,16 +147,22 @@ class Scanner:
             # Trim string from outer spaces to make future `x in str` checks faster
             target_line_stripped = target.line_strip
             target_line_stripped_len = target.line_strip_len
+            # use lower case for required substring
+            target_line_stripped_lower = target.line_lower_strip
 
             # "cache" - YAPF and pycharm formatters ...
             matched_keyword = \
                 target_line_stripped_len >= self.min_keyword_len and (  #
                         '=' in target_line_stripped
                         or ':' in target_line_stripped
-                        or "set" in target_line_stripped
-                        or "#define" in target_line_stripped
-                        or "%define" in target_line_stripped
+                        or ("define" in target_line_stripped
+                            and ('(' in target_line_stripped and ',' in target_line_stripped
+                                 or "#define" in target_line_stripped
+                                 or "%define" in target_line_stripped)
+                            )
                         or "%global" in target_line_stripped
+                        or "set" in target_line_stripped_lower
+                        or "%3d" in target_line_stripped_lower
                 )  #
             matched_pem_key = \
                 target_line_stripped_len >= self.min_pem_key_len \
@@ -165,8 +176,6 @@ class Scanner:
                              target.line_num)
                 continue
 
-            # use lower case for required substring
-            target_line_stripped_lower = target.line_lower_strip
             # cached value to skip the same regex verifying
             matched_regex: Dict[re.Pattern, bool] = {}
 

credsweeper/secret/config.json

@@ -12,18 +12,21 @@
             ".rpm",
             ".tar",
             ".war",
+            ".whl",
             ".xz",
             ".zip"
         ],
         "documents": [
-            ".xlsx",
+            ".doc",
             ".docx",
-            ".pptx",
-            ".xls",
             ".odp",
             ".ods",
             ".odt",
-            ".pdf"
+            ".pdf",
+            ".ppt",
+            ".pptx",
+            ".xls",
+            ".xlsx"
         ],
         "extension": [
             ".7z",
@@ -45,16 +48,23 @@
             ".info",
             ".jpeg",
             ".jpg",
+            ".lib",
             ".map",
             ".m4a",
             ".mat",
             ".mo",
+            ".mov",
             ".mp3",
             ".mp4",
+            ".mpg",
+            ".mkv",
             ".npy",
             ".npz",
             ".obj",
+            ".oga",
             ".ogg",
+            ".ogv",
+            ".ops",
             ".pak",
             ".png",
             ".psd",
@@ -71,8 +81,10 @@
             ".so",
             ".sum",
             ".svg",
+            ".swf",
             ".tif",
             ".tiff",
+            ".tlb",
             ".ttf",
             ".vcxproj",
             ".vdproj",
@@ -81,6 +93,7 @@
             ".webp",
             ".wma",
             ".woff",
+            ".woff2",
             ".yuv"
         ],
         "path": [
@@ -164,8 +177,8 @@
         "tizen"
     ],
     "check_for_literals": true,
-    "min_pattern_value_length": 12,
-    "min_keyword_value_length": 4,
+    "max_password_value_length": 64,
+    "max_url_cred_value_length": 80,
     "line_data_output": [
         "line",
         "line_num",

credsweeper/utils/__init__.py

@@ -1 +0,0 @@
-from credsweeper.utils.util import DiffRowData, Util, DiffDict

credsweeper/utils/pem_key_detector.py

@@ -5,10 +5,10 @@ import string
 from typing import List
 
 from credsweeper.common.constants import PEM_BEGIN_PATTERN, PEM_END_PATTERN, Chars
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/utils/util.py

@@ -9,12 +9,10 @@ import random
 import re
 import string
 import tarfile
-from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Dict, List, Tuple, Optional, Union
 
 import numpy as np
-import whatthepatch
 import yaml
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.asymmetric import padding
@@ -29,31 +27,12 @@ from cryptography.hazmat.primitives.asymmetric.x448 import X448PublicKey, X448Pr
 from cryptography.hazmat.primitives.serialization import load_der_private_key
 from cryptography.hazmat.primitives.serialization.pkcs12 import load_key_and_certificates
 from lxml import etree
-from typing_extensions import TypedDict
 
-from credsweeper.common.constants import DiffRowType, AVAILABLE_ENCODINGS, \
+from credsweeper.common.constants import AVAILABLE_ENCODINGS, \
     DEFAULT_ENCODING, LATIN_1, CHUNK_SIZE, MAX_LINE_LENGTH, CHUNK_STEP_SIZE, ASCII
 
 logger = logging.getLogger(__name__)
 
-DiffDict = TypedDict(
-    "DiffDict",
-    {
-        "old": Optional[int],  #
-        "new": Optional[int],  #
-        "line": Union[str, bytes],  # bytes are possibly since whatthepatch v1.0.4
-        "hunk": Any  # not used
-    })
-
-
-@dataclass(frozen=True)
-class DiffRowData:
-    """Class for keeping data of diff row."""
-
-    line_type: DiffRowType
-    line_numb: int
-    line: str
-
 
 class Util:
     """Class that contains different useful methods."""
@@ -82,11 +61,11 @@ class Util:
     def get_shannon_entropy(data: Union[str, bytes]) -> float:
         """Borrowed from http://blog.dkbza.org/2007/05/scanning-data-for-entropy-anomalies.html."""
         if not data:
-            return 0.
+            return 0.0
         size = len(data)
         _uniq, counts = np.unique(list(data), return_counts=True)
         probabilities = counts / size
-        entropy = float(-np.sum(probabilities * np.log2(probabilities)))
+        entropy = -float(np.sum(probabilities * np.log2(probabilities)))
         return entropy
 
     # Precalculated data for speedup
@@ -162,15 +141,6 @@ class Util:
         min_entropy = Util.get_min_data_entropy(data_len)
         return entropy < min_entropy
 
-    @staticmethod
-    def is_known(data: Union[bytes, bytearray]) -> bool:
-        """Returns True if any known binary format is found to prevent extra scan a file without an extension."""
-        if isinstance(data, (bytes, bytearray)) and data.startswith(b"\x7f\x45\x4c\x46") and 127 <= len(data):
-            # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format
-            # minimal ELF is 127 bytes https://github.com/tchajed/minimal-elf
-            return True
-        return False
-
     @staticmethod
     def is_binary(data: Union[bytes, bytearray]) -> bool:
         """
@@ -239,13 +209,12 @@ class Util:
             try:
                 if binary_suggest and LATIN_1 == encoding and (Util.is_binary(content) or not Util.is_latin1(content)):
                     # LATIN_1 may convert data (bytes in range 0x80:0xFF are transformed)
-                    # so skip this encoding when checking binaries
-                    logger.warning("Binary file detected %s", repr(content[:8]))
                     break
-                text = content.decode(encoding, errors="strict")
-                if content != text.encode(encoding, errors="strict"):
+                _text = content.decode(encoding=encoding, errors="strict")
+                if content != _text.encode(encoding=encoding, errors="strict"):
                     # the check helps to detect a real encoding
                     raise UnicodeError
+                text = _text
                 break
             except UnicodeError:
                 binary_suggest = True
@@ -254,6 +223,11 @@ class Util:
                 logger.error(f"Unexpected Error: Can't read content as {encoding}. Error message: {exc}")
         return text
 
+    @staticmethod
+    def split_text(text: str) -> List[str]:
+        """Splits a text into lines, handling all common line endings (e.g., LF, CRLF, CR)."""
+        return text.replace("\r\n", '\n').replace('\r', '\n').split('\n')
+
     @staticmethod
     def decode_bytes(content: bytes, encodings: Optional[List[str]] = None) -> List[str]:
         """Decode content using different encodings.
@@ -272,121 +246,11 @@ class Util:
 
         """
         if text := Util.decode_text(content, encodings):
-            lines = text.replace('\r\n', '\n').replace('\r', '\n').split('\n')
+            lines = Util.split_text(text)
         else:
             lines = []
         return lines
 
-    @staticmethod
-    def patch2files_diff(raw_patch: List[str], change_type: DiffRowType) -> Dict[str, List[DiffDict]]:
-        """Generate files changes from patch for added or deleted filepaths.
-
-        Args:
-            raw_patch: git patch file content
-            change_type: change type to select, DiffRowType.ADDED or DiffRowType.DELETED
-
-        Return:
-            return dict with ``{file paths: list of file row changes}``, where
-            elements of list of file row changes represented as::
-
-                {
-                    "old": line number before diff,
-                    "new": line number after diff,
-                    "line": line text,
-                    "hunk": diff hunk number
-                }
-
-        """
-        if not raw_patch:
-            return {}
-
-        added_files, deleted_files = {}, {}
-        try:
-            for patch in whatthepatch.parse_patch(raw_patch):
-                if patch.changes is None:
-                    logger.warning(f"Patch '{str(patch.header)}' cannot be scanned")
-                    continue
-                changes = []
-                for change in patch.changes:
-                    change_dict = change._asdict()
-                    changes.append(change_dict)
-
-                added_files[patch.header.new_path] = changes
-                deleted_files[patch.header.old_path] = changes
-            if change_type == DiffRowType.ADDED:
-                return added_files
-            elif change_type == DiffRowType.DELETED:
-                return deleted_files
-            else:
-                logger.error(f"Change type should be one of: '{DiffRowType.ADDED}', '{DiffRowType.DELETED}';"
-                             f" but received {change_type}")
-        except Exception as exc:
-            logger.exception(exc)
-        return {}
-
-    @staticmethod
-    def preprocess_diff_rows(
-            added_line_number: Optional[int],  #
-            deleted_line_number: Optional[int],  #
-            line: str) -> List[DiffRowData]:
-        """Auxiliary function to extend diff changes.
-
-        Args:
-            added_line_number: number of added line or None
-            deleted_line_number: number of deleted line or None
-            line: the text line
-
-        Return:
-            diff rows data with as list of row change type, line number, row content
-
-        """
-        rows_data: List[DiffRowData] = []
-        if isinstance(added_line_number, int):
-            # indicates line was inserted
-            rows_data.append(DiffRowData(DiffRowType.ADDED, added_line_number, line))
-        if isinstance(deleted_line_number, int):
-            # indicates line was removed
-            rows_data.append(DiffRowData(DiffRowType.DELETED, deleted_line_number, line))
-        return rows_data
-
-    @staticmethod
-    def wrong_change(change: DiffDict) -> bool:
-        """Returns True if the change is wrong"""
-        for i in ["line", "new", "old"]:
-            if i not in change:
-                logger.error(f"Skipping wrong change {change}")
-                return True
-        return False
-
-    @staticmethod
-    def preprocess_file_diff(changes: List[DiffDict]) -> List[DiffRowData]:
-        """Generate changed file rows from diff data with changed lines (e.g. marked + or - in diff).
-
-        Args:
-            changes: git diff by file rows data
-
-        Return:
-            diff rows data with as list of row change type, line number, row content
-
-        """
-        if not changes:
-            return []
-
-        rows_data = []
-        # process diff to restore lines and their positions
-        for change in changes:
-            if Util.wrong_change(change):
-                continue
-            line = change["line"]
-            if isinstance(line, str):
-                rows_data.extend(Util.preprocess_diff_rows(change.get("new"), change.get("old"), line))
-            elif isinstance(line, (bytes, bytearray)):
-                logger.warning("The feature is available with the deep scan option")
-            else:
-                logger.error(f"Unknown type of line {type(line)}")
-
-        return rows_data
-
     @staticmethod
     def is_zip(data: Union[bytes, bytearray]) -> bool:
         """According https://en.wikipedia.org/wiki/List_of_file_signatures"""
@@ -486,13 +350,20 @@ class Util:
             return True
         return False
 
-    @classmethod
-    def is_sqlite3(cls, data):
+    @staticmethod
+    def is_sqlite3(data: Union[bytes, bytearray]):
         """According https://en.wikipedia.org/wiki/List_of_file_signatures - SQLite Database"""
         if isinstance(data, (bytes, bytearray)) and data.startswith(b"SQLite format 3\0"):
             return True
         return False
 
+    @staticmethod
+    def is_rtf(data: Union[bytes, bytearray]):
+        """According https://en.wikipedia.org/wiki/List_of_file_signatures - Rich Text Format"""
+        if isinstance(data, (bytes, bytearray)) and data.startswith(b"{\\rtf1") and data.endswith(b"}"):
+            return True
+        return False
+
     @staticmethod
     def is_asn1(data: Union[bytes, bytearray]) -> int:
         """Only sequence type 0x30 and size correctness are checked
@@ -706,6 +577,7 @@ class Util:
         """decode text to bytes with / without padding detect and urlsafe symbols"""
         value = text.translate(Util.WHITESPACE_TRANS_TABLE)
         if padding_safe:
+            value = value.rstrip('=')  # python 3.10 workaround
             pad_num = 0x3 & len(value)
             if pad_num:
                 value += '=' * (4 - pad_num)
@@ -769,6 +641,8 @@ class Util:
     @staticmethod
     def subtext(text: str, pos: int, hunk_size: int) -> str:
         """cut text symmetrically for given position or use remained quota to be fitted in 2x hunk_size"""
+        # cut trailed whitespaces to obtain more informative data
+        text = text.rstrip()
         if hunk_size <= pos:
             left_quota = 0
             left_pos = pos - hunk_size

{credsweeper-1.11.5.dist-info → credsweeper-1.13.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.11.5
+Version: 1.13.3
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues
@@ -10,13 +10,12 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Security
 Classifier: Topic :: Software Development :: Quality Assurance
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: base58
 Requires-Dist: beautifulsoup4>=4.11.0
 Requires-Dist: colorama
@@ -24,10 +23,10 @@ Requires-Dist: cryptography
 Requires-Dist: gitpython
 Requires-Dist: humanfriendly
 Requires-Dist: lxml
-Requires-Dist: numpy<2.0.0
+Requires-Dist: numpy
 Requires-Dist: odfpy
-Requires-Dist: onnxruntime; platform_system != 'Windows'
-Requires-Dist: onnxruntime==1.19.2; platform_system == 'Windows'
+Requires-Dist: onnxruntime; platform_system != 'Windows' or python_version != '3.12'
+Requires-Dist: onnxruntime==1.19.2; platform_system == 'Windows' and python_version == '3.12'
 Requires-Dist: openpyxl
 Requires-Dist: pandas
 Requires-Dist: pdfminer-six
@@ -38,6 +37,7 @@ Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Requires-Dist: pyyaml
 Requires-Dist: rpmfile
+Requires-Dist: striprtf
 Requires-Dist: whatthepatch
 Requires-Dist: xlrd
 Description-Content-Type: text/markdown
@@ -90,7 +90,7 @@ Full documentation can be found here: <https://credsweeper.readthedocs.io/>
 
 ### Main Requirements
 
-- Python 3.9, 3.10, 3.11, 3.12
+- Python 3.10, 3.11, 3.12
 
 ### Installation