credsweeper 1.11.3__tar.gz → 1.11.4__tar.gz

{credsweeper-1.11.3 → credsweeper-1.11.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.11.3
+Version: 1.11.4
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.11.3"
+__version__ = "1.11.4"

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/__main__.py

@@ -63,7 +63,7 @@ def logger_levels(log_level: str) -> str:
     Returns True if log_level UPPERCASE is one of keys
     """
     val = log_level.upper()
-    if any(val == i for i in Logger.LEVELS.keys()):
+    if val in Logger.LEVELS:
         return val
     raise ArgumentTypeError(f"Log level provided: {log_level} -- must be one of: {' | '.join(Logger.LEVELS.keys())}")
 

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/app.py

@@ -15,11 +15,13 @@ from credsweeper.common.constants import Severity, ThresholdPreset, DiffRowType,
 from credsweeper.config import Config
 from credsweeper.credentials import Candidate, CredentialManager, CandidateKey
 from credsweeper.deep_scanner.deep_scanner import DeepScanner
+from credsweeper.file_handler.content_provider import ContentProvider
 from credsweeper.file_handler.diff_content_provider import DiffContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
 from credsweeper.file_handler.abstract_provider import AbstractProvider
 from credsweeper.file_handler.text_content_provider import TextContentProvider
 from credsweeper.scanner import Scanner
+from credsweeper.ml_model.ml_validator import MlValidator
 from credsweeper.utils import Util
 
 logger = logging.getLogger(__name__)
@@ -94,7 +96,7 @@ class CredSweeper:
             log_level: str - level for pool initializer according logging levels (UPPERCASE)
 
         """
-        self.pool_count: int = int(pool_count) if int(pool_count) > 1 else 1
+        self.pool_count: int = max(1, int(pool_count))
         if not (_severity := Severity.get(severity)):
             raise RuntimeError(f"Severity level provided: {severity}"
                                f" -- must be one of: {' | '.join([i.value for i in Severity])}")
@@ -123,9 +125,9 @@ class CredSweeper:
         self.ml_config = ml_config
         self.ml_model = ml_model
         self.ml_providers = ml_providers
-        self.ml_validator = None
         self.__thrifty = thrifty
         self.__log_level = log_level
+        self.__ml_validator: Optional[MlValidator] = None
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
@@ -182,35 +184,22 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    # the import cannot be done on top due
-    # TypeError: cannot pickle 'onnxruntime.capi.onnxruntime_pybind11_state.InferenceSession' object
-    from credsweeper.ml_model import MlValidator
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
     @property
     def ml_validator(self) -> MlValidator:
         """ml_validator getter"""
-        from credsweeper.ml_model import MlValidator
         if not self.__ml_validator:
-            self.__ml_validator: MlValidator = MlValidator(
+            self.__ml_validator = MlValidator(
                 threshold=self.ml_threshold,  #
                 ml_config=self.ml_config,  #
                 ml_model=self.ml_model,  #
                 ml_providers=self.ml_providers,  #
             )
-        assert self.__ml_validator, "self.__ml_validator was not initialized"
+        if not self.__ml_validator:
+            raise RuntimeError("MlValidator was not initialized!")
         return self.__ml_validator
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    @ml_validator.setter
-    def ml_validator(self, _ml_validator: Optional[MlValidator]) -> None:
-        """ml_validator setter"""
-        self.__ml_validator = _ml_validator
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
     @staticmethod
     def pool_initializer(log_kwargs) -> None:
         """Ignore SIGINT in child processes."""
@@ -219,20 +208,6 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    @property
-    def config(self) -> Config:
-        """config getter"""
-        return self.__config
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
-    @config.setter
-    def config(self, config: Config) -> None:
-        """config setter"""
-        self.__config = config
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
     def run(self, content_provider: AbstractProvider) -> int:
         """Run an analysis of 'content_provider' object.
 
@@ -241,9 +216,10 @@ class CredSweeper:
 
         """
         _empty_list: Sequence[Union[DiffContentProvider, TextContentProvider]] = []
-        file_extractors: Sequence[Union[DiffContentProvider, TextContentProvider]] = \
-            content_provider.get_scannable_files(self.config) if content_provider else _empty_list
-        logger.info(f"Start Scanner for {len(file_extractors)} providers")
+        file_extractors = content_provider.get_scannable_files(self.config) if content_provider else _empty_list
+        if not file_extractors:
+            logger.info(f"No scannable targets for {len(content_provider.paths)} paths")
+            return 0
         self.scan(file_extractors)
         self.post_processing()
         # PatchesProvider has the attribute. Circular import error appears with using the isinstance
@@ -260,7 +236,7 @@ class CredSweeper:
             content_providers: file objects to scan
 
         """
-        if 1 < self.pool_count:
+        if 1 < self.pool_count and 1 < len(content_providers):
             self.__multi_jobs_scan(content_providers)
         else:
             self.__single_job_scan(content_providers)
@@ -269,6 +245,7 @@ class CredSweeper:
 
     def __single_job_scan(self, content_providers: Sequence[Union[DiffContentProvider, TextContentProvider]]) -> None:
         """Performs scan in main thread"""
+        logger.info(f"Scan for {len(content_providers)} providers")
         all_cred = self.files_scan(content_providers)
         self.credential_manager.set_credentials(all_cred)
 
@@ -284,12 +261,14 @@ class CredSweeper:
             if "SILENCE" == self.__log_level:
                 logging.addLevelName(60, "SILENCE")
             log_kwargs["level"] = self.__log_level
-        with multiprocessing.get_context("spawn").Pool(processes=self.pool_count,
-                                                       initializer=self.pool_initializer,
+        pool_count = min(self.pool_count, len(content_providers))
+        logger.info(f"Scan in {pool_count} processes for {len(content_providers)} providers")
+        with multiprocessing.get_context("spawn").Pool(processes=pool_count,
+                                                       initializer=CredSweeper.pool_initializer,
                                                        initargs=(log_kwargs, )) as pool:
             try:
-                for scan_results in pool.imap_unordered(self.files_scan, (content_providers[x::self.pool_count]
-                                                                          for x in range(self.pool_count))):
+                for scan_results in pool.imap_unordered(self.files_scan,
+                                                        (content_providers[x::pool_count] for x in range(pool_count))):
                     for cred in scan_results:
                         self.credential_manager.add_credential(cred)
             except KeyboardInterrupt:
@@ -301,9 +280,7 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    def files_scan(
-            self,  #
-            content_providers: Sequence[Union[DiffContentProvider, TextContentProvider]]) -> List[Candidate]:
+    def files_scan(self, content_providers: Sequence[ContentProvider]) -> List[Candidate]:
         """Auxiliary method for scan one sequence"""
         all_cred: List[Candidate] = []
         for provider in content_providers:
@@ -316,7 +293,7 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    def file_scan(self, content_provider: Union[DiffContentProvider, TextContentProvider]) -> List[Candidate]:
+    def file_scan(self, content_provider: ContentProvider) -> List[Candidate]:
         """Run scanning of file from 'file_provider'.
 
         Args:

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/common/constants.py

@@ -96,10 +96,6 @@ class Chars(Enum):
     ASCII_PRINTABLE = string.printable
 
 
-ENTROPY_LIMIT_BASE64 = 4.5
-ENTROPY_LIMIT_BASE3x = 3
-
-
 class GroupType(Enum):
     """Group type - used in Group constructor for load predefined set of filters"""
     KEYWORD = "keyword"
@@ -148,7 +144,8 @@ OVERLAP_SIZE = 1000
 CHUNK_STEP_SIZE = CHUNK_SIZE - OVERLAP_SIZE
 # ML hunk size to limit of variable or value size and get substring near value
 ML_HUNK = 80
-""" values according https://docs.python.org/3/library/codecs.html """
+
+# values according https://docs.python.org/3/library/codecs.html
 UTF_8 = "utf_8"
 UTF_16 = "utf_16"
 LATIN_1 = "latin_1"

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/credentials/candidate_key.py

@@ -24,7 +24,7 @@ class CandidateKey:
         return self.key == other.key
 
     def __ne__(self, other):
-        return not (self == other)
+        return not bool(self == other)
 
     def __repr__(self) -> str:
         return f"{self.key}:{self.__line}"

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/credentials/credential_manager.py

@@ -110,7 +110,8 @@ class CredentialManager:
                 # Match by file path+line num+value. Value required so two different credentials still be
                 #  processed independently
                 candidate_key = CandidateKey(line_data)
-                if candidate_key not in groups:
-                    groups[candidate_key] = list()
-                groups[candidate_key].append(credential_candidate)
+                if candidate_key in groups:
+                    groups[candidate_key].append(credential_candidate)
+                else:
+                    groups[candidate_key] = [credential_candidate]
         return groups

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/credentials/line_data.py

@@ -327,11 +327,8 @@ class LineData:
             True if file require quotation, False otherwise
 
         """
-        if not self.path:
-            return False
-        if Util.get_extension(self.path) in self.config.source_quote_ext:
-            return True
-        return False
+        file_type = self.file_type or Util.get_extension(self.path)
+        return bool(file_type) and file_type in self.config.source_quote_ext
 
     @staticmethod
     def get_hash_or_subtext(

credsweeper-1.11.4/credsweeper/deep_scanner/deb_scanner.py

@@ -0,0 +1,48 @@
+import logging
+from abc import ABC
+from typing import List, Optional
+
+from credsweeper.common.constants import ASCII, MIN_DATA_LEN
+from credsweeper.credentials import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+
+logger = logging.getLogger(__name__)
+
+
+class DebScanner(AbstractScanner, ABC):
+    """Implements deb (ar) scanning"""
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Extracts data file from .ar (debian) archive and launches data_scan"""
+        candidates: Optional[List[Candidate]] = None
+        offset = 8  # b"!<arch>\n"
+        while offset < len(data_provider.data):
+            try:
+                file_size_data = data_provider.data[offset + 48:offset + 58]
+                file_size = int(file_size_data.decode(ASCII))
+                offset += 60
+                if file_size < MIN_DATA_LEN:
+                    offset += file_size
+                    continue
+                data = data_provider.data[offset:offset + file_size]
+                deb_content_provider = DataContentProvider(data=data,
+                                                           file_path=data_provider.file_path,
+                                                           file_type=data_provider.file_type,
+                                                           info=f"{data_provider.info}|DEB:0x{offset:x}")
+                new_limit = recursive_limit_size - file_size
+                deb_candidates = self.recursive_scan(deb_content_provider, depth, new_limit)
+                if deb_candidates is not None:
+                    if candidates:
+                        candidates.extend(deb_candidates)
+                    else:
+                        candidates = deb_candidates
+                # data padding = 2
+                offset += 1 + file_size if 1 & file_size else file_size
+            except Exception as exc:
+                logger.error(exc)
+        return candidates

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/deep_scanner/deep_scanner.py

@@ -1,8 +1,9 @@
+import contextlib
 import datetime
 import logging
 from typing import List, Optional, Any, Tuple, Union
 
-from credsweeper.common.constants import RECURSIVE_SCAN_LIMITATION, MIN_DATA_LEN
+from credsweeper.common.constants import RECURSIVE_SCAN_LIMITATION, MIN_DATA_LEN, MIN_VALUE_LENGTH
 from credsweeper.config import Config
 from credsweeper.credentials import Candidate
 from credsweeper.credentials.augment_candidates import augment_candidates
@@ -16,6 +17,7 @@ from credsweeper.scanner import Scanner
 from credsweeper.utils import Util
 from .byte_scanner import ByteScanner
 from .bzip2_scanner import Bzip2Scanner
+from .deb_scanner import DebScanner
 from .docx_scanner import DocxScanner
 from .eml_scanner import EmlScanner
 from .encoder_scanner import EncoderScanner
@@ -54,6 +56,7 @@ class DeepScanner(
     Pkcs12Scanner,  #
     PptxScanner,  #
     TarScanner,  #
+    DebScanner,  #
     XmlScanner,  #
     XlsxScanner,  #
     ZipScanner
@@ -114,6 +117,9 @@ class DeepScanner(
         elif Util.is_tar(data):
             if 0 < depth:
                 deep_scanners.append(TarScanner)
+        elif Util.is_deb(data):
+            if 0 < depth:
+                deep_scanners.append(DebScanner)
         elif Util.is_gzip(data):
             if 0 < depth:
                 deep_scanners.append(GzipScanner)
@@ -209,10 +215,10 @@ class DeepScanner(
                                                                   int) else RECURSIVE_SCAN_LIMITATION
         candidates: List[Candidate] = []
         data: Optional[bytes] = None
-        if isinstance(content_provider, TextContentProvider) or isinstance(content_provider, ByteContentProvider):
+        if isinstance(content_provider, (TextContentProvider, ByteContentProvider)):
             # Feature to scan files which might be containers
             data = content_provider.data
-            info = "FILE"
+            info = f"FILE:{content_provider.file_path}"
         elif isinstance(content_provider, DiffContentProvider) and content_provider.diff:
             candidates = self.scanner.scan(content_provider)
             # Feature to scan binary diffs
@@ -220,7 +226,7 @@ class DeepScanner(
             # the check for legal fix mypy issue
             if isinstance(diff, bytes):
                 data = diff
-            info = "DIFF"
+            info = f"DIFF:{content_provider.file_path}"
         else:
             logger.warning(f"Content provider {type(content_provider)} does not support deep scan")
             info = "NA"
@@ -298,7 +304,7 @@ class DeepScanner(
         items: List[Tuple[Union[int, str], Any]] = []
         struct_key: Optional[str] = None
         struct_value: Optional[str] = None
-        line_for_keyword_rules = ""
+        lines_for_keyword_rules = []
         if isinstance(struct_provider.struct, dict):
             for key, value in struct_provider.struct.items():
                 if isinstance(value, (list, tuple)) and 1 == len(value):
@@ -309,13 +315,13 @@ class DeepScanner(
             # for transformation {"key": "api_key", "value": "XXXXXXX"} -> {"api_key": "XXXXXXX"}
             struct_key = struct_provider.struct.get("key")
             struct_value = struct_provider.struct.get("value")
-        elif isinstance(struct_provider.struct, list) or isinstance(struct_provider.struct, tuple):
+        elif isinstance(struct_provider.struct, (list, tuple)):
             items = list(enumerate(struct_provider.struct))
         else:
             logger.error("Not supported type:%s val:%s", str(type(struct_provider.struct)), str(struct_provider.struct))
 
         for key, value in items:
-            if isinstance(value, dict) or isinstance(value, (list, tuple)) and 1 < len(value):
+            if isinstance(value, dict) or isinstance(value, (list, tuple)) and 1 <= len(value):
                 val_struct_provider = StructContentProvider(struct=value,
                                                             file_path=struct_provider.file_path,
                                                             file_type=struct_provider.file_type,
@@ -324,52 +330,57 @@ class DeepScanner(
                 candidates.extend(new_candidates)
 
             elif isinstance(value, bytes):
-                bytes_struct_provider = DataContentProvider(data=value,
-                                                            file_path=struct_provider.file_path,
-                                                            file_type=struct_provider.file_type,
-                                                            info=f"{struct_provider.info}|BYTES:{key}")
-                new_limit = recursive_limit_size - len(value)
-                new_candidates = self.recursive_scan(bytes_struct_provider, depth, new_limit)
-                candidates.extend(new_candidates)
+                if MIN_DATA_LEN <= len(value):
+                    bytes_struct_provider = DataContentProvider(data=value,
+                                                                file_path=struct_provider.file_path,
+                                                                file_type=struct_provider.file_type,
+                                                                info=f"{struct_provider.info}|BYTES:{key}")
+                    new_limit = recursive_limit_size - len(value)
+                    new_candidates = self.recursive_scan(bytes_struct_provider, depth, new_limit)
+                    candidates.extend(new_candidates)
+                if MIN_VALUE_LENGTH <= len(value) and isinstance(key, str) \
+                        and self.scanner.keywords_required_substrings_check(key.lower()):
+                    str_val = str(value)
+                    lines_for_keyword_rules.append(f"{key} = '{str_val}'" if '"' in str_val else f'{key} = "{str_val}"')
 
             elif isinstance(value, str):
-                data = value.encode(encoding=DEFAULT_ENCODING, errors='replace')
-                str_struct_provider = DataContentProvider(data=data,
-                                                          file_path=struct_provider.file_path,
-                                                          file_type=struct_provider.file_type,
-                                                          info=f"{struct_provider.info}|STRING:{key}")
-                new_limit = recursive_limit_size - len(str_struct_provider.data)
-                new_candidates = self.recursive_scan(str_struct_provider, depth, new_limit)
-                candidates.extend(new_candidates)
-
+                if MIN_DATA_LEN <= len(value):
+                    # recursive scan only for data which may be decoded at least
+                    with contextlib.suppress(UnicodeError):
+                        data = value.encode(encoding=DEFAULT_ENCODING, errors='strict')
+                        str_struct_provider = DataContentProvider(data=data,
+                                                                  file_path=struct_provider.file_path,
+                                                                  file_type=struct_provider.file_type,
+                                                                  info=f"{struct_provider.info}|STRING:{key}")
+                        new_limit = recursive_limit_size - len(str_struct_provider.data)
+                        new_candidates = self.recursive_scan(str_struct_provider, depth, new_limit)
+                        candidates.extend(new_candidates)
                 # use key = "value" scan for common cases like in TOML
-                if isinstance(key, str) and self.scanner.keywords_required_substrings_check(key):
-                    line_for_keyword_rules += f"{key} = \"{value}\"; "
+                if MIN_VALUE_LENGTH <= len(value) and isinstance(key, str) \
+                        and self.scanner.keywords_required_substrings_check(key.lower()):
+                    lines_for_keyword_rules.append(f"{key} = '{value}'" if '"' in value else f'{key} = "{value}"')
 
             elif isinstance(value, (int, float, datetime.date, datetime.datetime)):
-                # use the fields only in case of matched keywords
-                if isinstance(key, str) and self.scanner.keywords_required_substrings_check(key):
-                    line_for_keyword_rules += f"{key} = \"{value}\"; "
-
+                # skip useless types
+                pass
             else:
                 logger.warning("Not supported type:%s value(%s)", str(type(value)), str(value))
 
-        if line_for_keyword_rules:
-            str_provider = StringContentProvider([line_for_keyword_rules],
+        if lines_for_keyword_rules:
+            str_provider = StringContentProvider(lines_for_keyword_rules,
                                                  file_path=struct_provider.file_path,
-                                                 file_type=".toml",
-                                                 info=f"{struct_provider.info}|KEYWORD:`{line_for_keyword_rules}`")
+                                                 file_type=".py",
+                                                 info=f"{struct_provider.info}|KEYWORD:`{lines_for_keyword_rules}`")
             new_candidates = self.scanner.scan(str_provider)
             augment_candidates(candidates, new_candidates)
 
         # last check when dictionary is {"key": "api_key", "value": "XXXXXXX"} -> {"api_key": "XXXXXXX"}
         if isinstance(struct_key, str) and isinstance(struct_value, str):
-            line_for_keyword_rules = f"{struct_key} = \"{struct_value}\""
             key_value_provider = StringContentProvider(
-                [line_for_keyword_rules],
+                [f"{struct_key} = '{struct_value}'" if '"' in struct_value else f'{struct_key} = "{struct_value}"'],
                 file_path=struct_provider.file_path,
                 file_type=".toml",
-                info=f"{struct_provider.info}|KEY_VALUE:`{line_for_keyword_rules}`")
+                info=f"{struct_provider.info}|KEY_VALUE:`{lines_for_keyword_rules}`")
             new_candidates = self.scanner.scan(key_value_provider)
             augment_candidates(candidates, new_candidates)
         return candidates

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/deep_scanner/gzip_scanner.py

@@ -31,7 +31,7 @@ class GzipScanner(AbstractScanner, ABC):
                 gzip_content_provider = DataContentProvider(data=f.read(),
                                                             file_path=new_path,
                                                             file_type=Util.get_extension(new_path),
-                                                            info=f"{data_provider.info}|GZIP:{file_path}")
+                                                            info=f"{data_provider.info}|GZIP:{new_path}")
                 new_limit = recursive_limit_size - len(gzip_content_provider.data)
                 gzip_candidates = self.recursive_scan(gzip_content_provider, depth, new_limit)
                 return gzip_candidates

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/byte_content_provider.py

@@ -32,10 +32,10 @@ class ByteContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__data = None
-        if hasattr(self, "data"):
+        if "data" in self.__dict__:
             delattr(self, "data")
         self.__lines = None
-        if hasattr(self, "lines"):
+        if "lines" in self.__dict__:
             delattr(self, "lines")
 
     @cached_property

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/content_provider.py

@@ -93,7 +93,7 @@ class ContentProvider(ABC):
             if min_len > len(line.strip()):
                 # Ignore target if stripped part is too short for all types
                 continue
-            elif MAX_LINE_LENGTH < len(line):
+            if MAX_LINE_LENGTH < len(line):
                 for chunk_start, chunk_end in Util.get_chunks(len(line)):
                     target = AnalysisTarget(
                         line_pos=line_pos,  #

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/data_content_provider.py

@@ -54,10 +54,10 @@ class DataContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__data = None
-        if hasattr(self, "data"):
+        if "data" in self.__dict__:
             delattr(self, "data")
         self.__text = None
-        if hasattr(self, "text"):
+        if "text" in self.__dict__:
             delattr(self, "text")
         self.structure = None
         self.decoded = None

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/diff_content_provider.py

@@ -48,8 +48,8 @@ class DiffContentProvider(ContentProvider):
 
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
-        self.__diff = None
-        if hasattr(self, "diff"):
+        self.__diff = []
+        if "diff" in self.__dict__:
             delattr(self, "diff")
 
     @staticmethod

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/file_path_extractor.py

@@ -162,7 +162,7 @@ class FilePathExtractor:
             True when the file is oversize or less than MIN_DATA_LEN, or unsupported
         """
         path = reference[1] if isinstance(reference, tuple) else reference
-        if isinstance(path, str) or isinstance(path, Path):
+        if isinstance(path, (str, Path)):
             file_size = os.path.getsize(path)
         elif isinstance(path, io.BytesIO):
             current_pos = path.tell()

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/files_provider.py

@@ -42,7 +42,7 @@ class FilesProvider(AbstractProvider):
         """
         text_content_provider_list: List[Union[DiffContentProvider, TextContentProvider]] = []
         for path in self.paths:
-            if isinstance(path, str) or isinstance(path, Path):
+            if isinstance(path, (str, Path)):
                 new_files = FilePathExtractor.get_file_paths(config, path)
                 if self.skip_ignored:
                     new_files = FilePathExtractor.apply_gitignore(new_files)
@@ -50,9 +50,7 @@ class FilesProvider(AbstractProvider):
                     text_content_provider_list.append(TextContentProvider(_file))
             elif isinstance(path, io.BytesIO):
                 text_content_provider_list.append(TextContentProvider((":memory:", path)))
-            elif isinstance(path, tuple) \
-                    and (isinstance(path[0], str) or isinstance(path[0], Path)) \
-                    and isinstance(path[1], io.BytesIO):
+            elif isinstance(path, tuple) and (isinstance(path[0], (str, Path))) and isinstance(path[1], io.BytesIO):
                 # suppose, all the files must be scanned
                 text_content_provider_list.append(TextContentProvider(path))
             else:

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/patches_provider.py

@@ -37,7 +37,7 @@ class PatchesProvider(AbstractProvider):
         for file_path in self.paths:
             if FilePathExtractor.check_file_size(config, file_path):
                 continue
-            if isinstance(file_path, str) or isinstance(file_path, Path):
+            if isinstance(file_path, (str, Path)):
                 raw_patches.append(Util.read_file(file_path))
             elif isinstance(file_path, io.BytesIO):
                 the_patch = Util.decode_bytes(file_path.read())

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/string_content_provider.py

@@ -38,10 +38,10 @@ class StringContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__lines = []
-        if hasattr(self, "lines"):
+        if "lines" in self.__dict__:
             delattr(self, "lines")
         self.__line_numbers = []
-        if hasattr(self, "line_numbers"):
+        if "line_numbers" in self.__dict__:
             delattr(self, "line_numbers")
 
     @cached_property

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/struct_content_provider.py

@@ -38,7 +38,7 @@ class StructContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__struct = None
-        if hasattr(self, "struct"):
+        if "struct" in self.__dict__:
             delattr(self, "struct")
 
     def yield_analysis_target(self, min_len: int) -> Generator[AnalysisTarget, None, None]:

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/file_handler/text_content_provider.py

@@ -42,10 +42,10 @@ class TextContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__data = None
-        if hasattr(self, "data"):
+        if "data" in self.__dict__:
             delattr(self, "data")
         self.__lines = None
-        if hasattr(self, "lines"):
+        if "lines" in self.__dict__:
             delattr(self, "lines")
         if isinstance(self.__io, io.BytesIO) and self.__io and not self.__io.closed:
             self.__io.close()

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/filters/value_base64_encoded_pem_check.py

@@ -30,7 +30,7 @@ class ValueBase64EncodedPem(Filter):
         with contextlib.suppress(Exception):
             text = Util.decode_base64(line_data.value, padding_safe=True, urlsafe_detect=True)
             lines = text.decode(ASCII).splitlines()
-            lines_pos = [x for x in range(len(lines))]
+            lines_pos = list(range(len(lines)))
             for line_pos, line in zip(lines_pos, lines):
                 if PEM_BEGIN_PATTERN in line:
                     new_target = AnalysisTarget(line_pos, lines, lines_pos, target.descriptor)

{credsweeper-1.11.3 → credsweeper-1.11.4}/credsweeper/filters/value_entropy_base64_check.py

@@ -19,12 +19,8 @@ class ValueEntropyBase64Check(ValueEntropyBaseCheck):
             y = 0.944 * math.log2(x) - 0.009 * x - 0.04
         elif 65 <= x < 256:
             y = 0.621 * math.log2(x) - 0.003 * x + 1.54
-        elif 256 <= x < 512:
-            y = 5.77
-        elif 512 <= x < 1024:
-            y = 5.89
-        elif 1024 <= x:
-            y = 5.94
+        elif 256 <= x:
+            y = 6 - 64 / x
         else:
             y = 0
         return y