credsweeper 1.11.3__tar.gz → 1.11.5__tar.gz

{credsweeper-1.11.3 → credsweeper-1.11.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.11.3
+Version: 1.11.5
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues
@@ -37,6 +37,7 @@ Requires-Dist: python-dateutil
 Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Requires-Dist: pyyaml
+Requires-Dist: rpmfile
 Requires-Dist: whatthepatch
 Requires-Dist: xlrd
 Description-Content-Type: text/markdown
@@ -140,11 +141,7 @@ cat output.json
                 "value_start": 12,
                 "value_end": 19,
                 "variable": "password",
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 2.120589933192232,
-                    "valid": false
-                }
+                "entropy": 2.12059
             }
         ]
     }

{credsweeper-1.11.3 → credsweeper-1.11.5}/README.md

@@ -97,11 +97,7 @@ cat output.json
                 "value_start": 12,
                 "value_end": 19,
                 "variable": "password",
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 2.120589933192232,
-                    "valid": false
-                }
+                "entropy": 2.12059
             }
         ]
     }

{credsweeper-1.11.3 → credsweeper-1.11.5}/credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.11.3"
+__version__ = "1.11.5"

{credsweeper-1.11.3 → credsweeper-1.11.5}/credsweeper/__main__.py

@@ -63,7 +63,7 @@ def logger_levels(log_level: str) -> str:
     Returns True if log_level UPPERCASE is one of keys
     """
     val = log_level.upper()
-    if any(val == i for i in Logger.LEVELS.keys()):
+    if val in Logger.LEVELS:
         return val
     raise ArgumentTypeError(f"Log level provided: {log_level} -- must be one of: {' | '.join(Logger.LEVELS.keys())}")
 

{credsweeper-1.11.3 → credsweeper-1.11.5}/credsweeper/app.py

@@ -15,11 +15,13 @@ from credsweeper.common.constants import Severity, ThresholdPreset, DiffRowType,
 from credsweeper.config import Config
 from credsweeper.credentials import Candidate, CredentialManager, CandidateKey
 from credsweeper.deep_scanner.deep_scanner import DeepScanner
+from credsweeper.file_handler.content_provider import ContentProvider
 from credsweeper.file_handler.diff_content_provider import DiffContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
 from credsweeper.file_handler.abstract_provider import AbstractProvider
 from credsweeper.file_handler.text_content_provider import TextContentProvider
 from credsweeper.scanner import Scanner
+from credsweeper.ml_model.ml_validator import MlValidator
 from credsweeper.utils import Util
 
 logger = logging.getLogger(__name__)
@@ -94,7 +96,7 @@ class CredSweeper:
             log_level: str - level for pool initializer according logging levels (UPPERCASE)
 
         """
-        self.pool_count: int = int(pool_count) if int(pool_count) > 1 else 1
+        self.pool_count: int = max(1, int(pool_count))
         if not (_severity := Severity.get(severity)):
             raise RuntimeError(f"Severity level provided: {severity}"
                                f" -- must be one of: {' | '.join([i.value for i in Severity])}")
@@ -123,9 +125,9 @@ class CredSweeper:
         self.ml_config = ml_config
         self.ml_model = ml_model
         self.ml_providers = ml_providers
-        self.ml_validator = None
         self.__thrifty = thrifty
         self.__log_level = log_level
+        self.__ml_validator: Optional[MlValidator] = None
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
@@ -182,35 +184,22 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    # the import cannot be done on top due
-    # TypeError: cannot pickle 'onnxruntime.capi.onnxruntime_pybind11_state.InferenceSession' object
-    from credsweeper.ml_model import MlValidator
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
     @property
     def ml_validator(self) -> MlValidator:
         """ml_validator getter"""
-        from credsweeper.ml_model import MlValidator
         if not self.__ml_validator:
-            self.__ml_validator: MlValidator = MlValidator(
+            self.__ml_validator = MlValidator(
                 threshold=self.ml_threshold,  #
                 ml_config=self.ml_config,  #
                 ml_model=self.ml_model,  #
                 ml_providers=self.ml_providers,  #
             )
-        assert self.__ml_validator, "self.__ml_validator was not initialized"
+        if not self.__ml_validator:
+            raise RuntimeError("MlValidator was not initialized!")
         return self.__ml_validator
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    @ml_validator.setter
-    def ml_validator(self, _ml_validator: Optional[MlValidator]) -> None:
-        """ml_validator setter"""
-        self.__ml_validator = _ml_validator
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
     @staticmethod
     def pool_initializer(log_kwargs) -> None:
         """Ignore SIGINT in child processes."""
@@ -219,20 +208,6 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    @property
-    def config(self) -> Config:
-        """config getter"""
-        return self.__config
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
-    @config.setter
-    def config(self, config: Config) -> None:
-        """config setter"""
-        self.__config = config
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
     def run(self, content_provider: AbstractProvider) -> int:
         """Run an analysis of 'content_provider' object.
 
@@ -241,9 +216,10 @@ class CredSweeper:
 
         """
         _empty_list: Sequence[Union[DiffContentProvider, TextContentProvider]] = []
-        file_extractors: Sequence[Union[DiffContentProvider, TextContentProvider]] = \
-            content_provider.get_scannable_files(self.config) if content_provider else _empty_list
-        logger.info(f"Start Scanner for {len(file_extractors)} providers")
+        file_extractors = content_provider.get_scannable_files(self.config) if content_provider else _empty_list
+        if not file_extractors:
+            logger.info(f"No scannable targets for {len(content_provider.paths)} paths")
+            return 0
         self.scan(file_extractors)
         self.post_processing()
         # PatchesProvider has the attribute. Circular import error appears with using the isinstance
@@ -260,7 +236,7 @@ class CredSweeper:
             content_providers: file objects to scan
 
         """
-        if 1 < self.pool_count:
+        if 1 < self.pool_count and 1 < len(content_providers):
             self.__multi_jobs_scan(content_providers)
         else:
             self.__single_job_scan(content_providers)
@@ -269,6 +245,7 @@ class CredSweeper:
 
     def __single_job_scan(self, content_providers: Sequence[Union[DiffContentProvider, TextContentProvider]]) -> None:
         """Performs scan in main thread"""
+        logger.info(f"Scan for {len(content_providers)} providers")
         all_cred = self.files_scan(content_providers)
         self.credential_manager.set_credentials(all_cred)
 
@@ -284,12 +261,14 @@ class CredSweeper:
             if "SILENCE" == self.__log_level:
                 logging.addLevelName(60, "SILENCE")
             log_kwargs["level"] = self.__log_level
-        with multiprocessing.get_context("spawn").Pool(processes=self.pool_count,
-                                                       initializer=self.pool_initializer,
+        pool_count = min(self.pool_count, len(content_providers))
+        logger.info(f"Scan in {pool_count} processes for {len(content_providers)} providers")
+        with multiprocessing.get_context("spawn").Pool(processes=pool_count,
+                                                       initializer=CredSweeper.pool_initializer,
                                                        initargs=(log_kwargs, )) as pool:
             try:
-                for scan_results in pool.imap_unordered(self.files_scan, (content_providers[x::self.pool_count]
-                                                                          for x in range(self.pool_count))):
+                for scan_results in pool.imap_unordered(self.files_scan,
+                                                        (content_providers[x::pool_count] for x in range(pool_count))):
                     for cred in scan_results:
                         self.credential_manager.add_credential(cred)
             except KeyboardInterrupt:
@@ -301,9 +280,7 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    def files_scan(
-            self,  #
-            content_providers: Sequence[Union[DiffContentProvider, TextContentProvider]]) -> List[Candidate]:
+    def files_scan(self, content_providers: Sequence[ContentProvider]) -> List[Candidate]:
         """Auxiliary method for scan one sequence"""
         all_cred: List[Candidate] = []
         for provider in content_providers:
@@ -316,7 +293,7 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    def file_scan(self, content_provider: Union[DiffContentProvider, TextContentProvider]) -> List[Candidate]:
+    def file_scan(self, content_provider: ContentProvider) -> List[Candidate]:
         """Run scanning of file from 'file_provider'.
 
         Args:

{credsweeper-1.11.3 → credsweeper-1.11.5}/credsweeper/common/constants.py

@@ -96,10 +96,6 @@ class Chars(Enum):
     ASCII_PRINTABLE = string.printable
 
 
-ENTROPY_LIMIT_BASE64 = 4.5
-ENTROPY_LIMIT_BASE3x = 3
-
-
 class GroupType(Enum):
     """Group type - used in Group constructor for load predefined set of filters"""
     KEYWORD = "keyword"
@@ -148,7 +144,8 @@ OVERLAP_SIZE = 1000
 CHUNK_STEP_SIZE = CHUNK_SIZE - OVERLAP_SIZE
 # ML hunk size to limit of variable or value size and get substring near value
 ML_HUNK = 80
-""" values according https://docs.python.org/3/library/codecs.html """
+
+# values according https://docs.python.org/3/library/codecs.html
 UTF_8 = "utf_8"
 UTF_16 = "utf_16"
 LATIN_1 = "latin_1"

{credsweeper-1.11.3 → credsweeper-1.11.5}/credsweeper/credentials/candidate_key.py

@@ -24,7 +24,7 @@ class CandidateKey:
         return self.key == other.key
 
     def __ne__(self, other):
-        return not (self == other)
+        return not bool(self == other)
 
     def __repr__(self) -> str:
         return f"{self.key}:{self.__line}"

{credsweeper-1.11.3 → credsweeper-1.11.5}/credsweeper/credentials/credential_manager.py

@@ -110,7 +110,8 @@ class CredentialManager:
                 # Match by file path+line num+value. Value required so two different credentials still be
                 #  processed independently
                 candidate_key = CandidateKey(line_data)
-                if candidate_key not in groups:
-                    groups[candidate_key] = list()
-                groups[candidate_key].append(credential_candidate)
+                if candidate_key in groups:
+                    groups[candidate_key].append(credential_candidate)
+                else:
+                    groups[candidate_key] = [credential_candidate]
         return groups

{credsweeper-1.11.3 → credsweeper-1.11.5}/credsweeper/credentials/line_data.py

@@ -327,11 +327,8 @@ class LineData:
             True if file require quotation, False otherwise
 
         """
-        if not self.path:
-            return False
-        if Util.get_extension(self.path) in self.config.source_quote_ext:
-            return True
-        return False
+        file_type = self.file_type or Util.get_extension(self.path)
+        return bool(file_type) and file_type in self.config.source_quote_ext
 
     @staticmethod
     def get_hash_or_subtext(

credsweeper-1.11.5/credsweeper/deep_scanner/abstract_scanner.py

@@ -0,0 +1,306 @@
+import contextlib
+import datetime
+import logging
+from abc import abstractmethod, ABC
+from typing import List, Optional, Tuple, Any, Generator
+
+from credsweeper.common.constants import RECURSIVE_SCAN_LIMITATION, MIN_DATA_LEN, DEFAULT_ENCODING, UTF_8, \
+    MIN_VALUE_LENGTH
+from credsweeper.config import Config
+from credsweeper.credentials import Candidate
+from credsweeper.credentials.augment_candidates import augment_candidates
+from credsweeper.file_handler.byte_content_provider import ByteContentProvider
+from credsweeper.file_handler.content_provider import ContentProvider
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.descriptor import Descriptor
+from credsweeper.file_handler.diff_content_provider import DiffContentProvider
+from credsweeper.file_handler.file_path_extractor import FilePathExtractor
+from credsweeper.file_handler.string_content_provider import StringContentProvider
+from credsweeper.file_handler.struct_content_provider import StructContentProvider
+from credsweeper.file_handler.text_content_provider import TextContentProvider
+from credsweeper.scanner import Scanner
+
+logger = logging.getLogger(__name__)
+
+
+class AbstractScanner(ABC):
+    """Base abstract class for all recursive scanners"""
+
+    @property
+    @abstractmethod
+    def config(self) -> Config:
+        """Abstract property to be defined in DeepScanner"""
+        raise NotImplementedError(__name__)
+
+    @property
+    @abstractmethod
+    def scanner(self) -> Scanner:
+        """Abstract property to be defined in DeepScanner"""
+        raise NotImplementedError(__name__)
+
+    @abstractmethod
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Abstract method to be defined in DeepScanner"""
+        raise NotImplementedError(__name__)
+
+    @staticmethod
+    @abstractmethod
+    def get_deep_scanners(data: bytes, descriptor: Descriptor, depth: int) -> Tuple[List[Any], List[Any]]:
+        """Returns possibly scan methods for the data depends on content and fallback scanners"""
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    def recursive_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int = 0,  #
+            recursive_limit_size: int = 0) -> List[Candidate]:
+        """Recursive function to scan files which might be containers like ZIP archives
+
+            Args:
+                data_provider: DataContentProvider object may be a container
+                depth: maximal level of recursion
+                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
+        """
+        candidates: List[Candidate] = []
+        if 0 > depth:
+            # break recursion if maximal depth is reached
+            logger.debug("Bottom reached %s recursive_limit_size:%d", data_provider.file_path, recursive_limit_size)
+            return candidates
+        depth -= 1
+        if MIN_DATA_LEN > len(data_provider.data):
+            # break recursion for minimal data size
+            logger.debug("Too small data: size=%d, depth=%d, limit=%d, path=%s, info=%s", len(data_provider.data),
+                         depth, recursive_limit_size, data_provider.file_path, data_provider.info)
+            return candidates
+        logger.debug("Start data_scan: size=%d, depth=%d, limit=%d, path=%s, info=%s", len(data_provider.data), depth,
+                     recursive_limit_size, data_provider.file_path, data_provider.info)
+
+        if FilePathExtractor.is_find_by_ext_file(self.config, data_provider.file_type):
+            # Skip scanning file and makes fake candidate due the extension is suspicious
+            dummy_candidate = Candidate.get_dummy_candidate(self.config, data_provider.file_path,
+                                                            data_provider.file_type, data_provider.info,
+                                                            FilePathExtractor.FIND_BY_EXT_RULE)
+            candidates.append(dummy_candidate)
+        else:
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size)
+            augment_candidates(candidates, new_candidates)
+
+        return candidates
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    @staticmethod
+    def key_value_combination(structure: dict) -> Generator[Tuple[Any, Any], None, None]:
+        """Combine items by `key` and `value` from a dictionary for augmentation
+        {..., "key": "api_key", "value": "XXXXXXX", ...} -> ("api_key", "XXXXXXX")
+
+        """
+        for key_id in ("key", "KEY", "Key"):
+            if key_id in structure:
+                struct_key = structure.get(key_id)
+                break
+        else:
+            struct_key = None
+        if isinstance(struct_key, bytes):
+            # sqlite table may produce bytes for `key`
+            with contextlib.suppress(UnicodeError):
+                struct_key = struct_key.decode(UTF_8)
+        # only str type is common used for the augmentation
+        if struct_key and isinstance(struct_key, str):
+            for value_id in ("value", "VALUE", "Value"):
+                if value_id in structure:
+                    struct_value = structure.get(value_id)
+                    if struct_value and isinstance(struct_value, (str, bytes)):
+                        yield struct_key, struct_value
+                        # break in successful case
+                        break
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    @staticmethod
+    def structure_processing(structure: Any) -> Generator[Tuple[Any, Any], None, None]:
+        """Yields pair `key, value` from given structure if applicable"""
+        if isinstance(structure, dict):
+            # transform dictionary to list
+            for key, value in structure.items():
+                if not value:
+                    # skip empty values
+                    continue
+                if isinstance(value, (list, tuple)):
+                    if 1 == len(value):
+                        # simplify some structures like YAML when single item in new line is a value
+                        yield key, value[0]
+                        continue
+                # all other data will be precessed in next code
+                yield key, value
+            yield from AbstractScanner.key_value_combination(structure)
+        elif isinstance(structure, (list, tuple)):
+            # enumerate the items to fit for return structure
+            for key, value in enumerate(structure):
+                yield key, value
+        else:
+            logger.error("Not supported type:%s val:%s", str(type(structure)), repr(structure))
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    def structure_scan(
+            self,  #
+            struct_provider: StructContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> List[Candidate]:
+        """Recursive function to scan structured data
+
+            Args:
+                struct_provider: DataContentProvider object may be a container
+                depth: maximal level of recursion
+                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
+        """
+        candidates: List[Candidate] = []
+        logger.debug("Start struct_scan: depth=%d, limit=%d, path=%s, info=%s", depth, recursive_limit_size,
+                     struct_provider.file_path, struct_provider.info)
+
+        if 0 > depth:
+            # break recursion if maximal depth is reached
+            logger.debug("bottom reached %s recursive_limit_size:%d", struct_provider.file_path, recursive_limit_size)
+            return candidates
+
+        depth -= 1
+
+        augmented_lines_for_keyword_rules = []
+        for key, value in AbstractScanner.structure_processing(struct_provider.struct):
+            # a keyword rule may be applicable for `key` (str only) and `value` (str, bytes)
+            keyword_match = bool(isinstance(key, str) and self.scanner.keywords_required_substrings_check(key.lower()))
+
+            if isinstance(value, (dict, list, tuple)) and value:
+                # recursive scan for not empty structured `value`
+                val_struct_provider = StructContentProvider(struct=value,
+                                                            file_path=struct_provider.file_path,
+                                                            file_type=struct_provider.file_type,
+                                                            info=f"{struct_provider.info}|STRUCT:{key}")
+                new_candidates = self.structure_scan(val_struct_provider, depth, recursive_limit_size)
+                candidates.extend(new_candidates)
+            elif isinstance(value, bytes):
+                # recursive data scan
+                if MIN_DATA_LEN <= len(value):
+                    bytes_struct_provider = DataContentProvider(data=value,
+                                                                file_path=struct_provider.file_path,
+                                                                file_type=struct_provider.file_type,
+                                                                info=f"{struct_provider.info}|BYTES:{key}")
+                    new_limit = recursive_limit_size - len(value)
+                    new_candidates = self.recursive_scan(bytes_struct_provider, depth, new_limit)
+                    candidates.extend(new_candidates)
+                if keyword_match and MIN_VALUE_LENGTH <= len(value):
+                    augmented_lines_for_keyword_rules.append(f"{key} = {repr(value)}")
+            elif isinstance(value, str):
+                # recursive text scan with transformation into bytes
+                stripped_value = value.strip()
+                if MIN_DATA_LEN <= len(stripped_value):
+                    # recursive scan only for data which may be decoded at least
+                    with contextlib.suppress(UnicodeError):
+                        data = stripped_value.encode(encoding=DEFAULT_ENCODING, errors='strict')
+                        str_struct_provider = DataContentProvider(data=data,
+                                                                  file_path=struct_provider.file_path,
+                                                                  file_type=struct_provider.file_type,
+                                                                  info=f"{struct_provider.info}|STRING:{key}")
+                        new_limit = recursive_limit_size - len(str_struct_provider.data)
+                        new_candidates = self.recursive_scan(str_struct_provider, depth, new_limit)
+                        candidates.extend(new_candidates)
+                if keyword_match and MIN_VALUE_LENGTH <= len(stripped_value):
+                    augmented_lines_for_keyword_rules.append(f"{key} = {repr(stripped_value)}")
+            elif value is None or isinstance(value, (int, float, datetime.date, datetime.datetime)):
+                # skip useless types
+                pass
+            else:
+                logger.warning("Not supported type:%s value(%s)", str(type(value)), str(value))
+
+        if augmented_lines_for_keyword_rules:
+            str_provider = StringContentProvider(augmented_lines_for_keyword_rules,
+                                                 file_path=struct_provider.file_path,
+                                                 file_type=struct_provider.file_type,
+                                                 info=f"{struct_provider.info}|KEYWORD")
+            new_candidates = self.scanner.scan(str_provider)
+            augment_candidates(candidates, new_candidates)
+
+        return candidates
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    def deep_scan_with_fallback(self, data_provider: DataContentProvider, depth: int,
+                                recursive_limit_size: int) -> List[Candidate]:
+        """Scans with deep scanners and fallback scanners if possible
+
+            Args:
+                data_provider: DataContentProvider with raw data
+                depth: maximal level of recursion
+                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
+
+            Returns: list with candidates
+
+        """
+        candidates: List[Candidate] = []
+        deep_scanners, fallback_scanners = self.get_deep_scanners(data_provider.data, data_provider.descriptor, depth)
+        fallback = True
+        for scan_class in deep_scanners:
+            new_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+            if new_candidates is None:
+                # scanner did not recognise the content type
+                continue
+            augment_candidates(candidates, new_candidates)
+            # this scan is successful, so fallback is not necessary
+            fallback = False
+        if fallback:
+            for scan_class in fallback_scanners:
+                fallback_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
+                if fallback_candidates is None:
+                    continue
+                augment_candidates(candidates, fallback_candidates)
+                # use only first successful fallback scanner
+                break
+        return candidates
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    def scan(self,
+             content_provider: ContentProvider,
+             depth: int,
+             recursive_limit_size: Optional[int] = None) -> List[Candidate]:
+        """Initial scan method to launch recursive scan. Skips ByteScanner to prevent extra scan
+
+            Args:
+                content_provider: ContentProvider that might contain raw data
+                depth: maximal level of recursion
+                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
+        """
+        recursive_limit_size = recursive_limit_size if isinstance(recursive_limit_size,
+                                                                  int) else RECURSIVE_SCAN_LIMITATION
+        candidates: List[Candidate] = []
+        data: Optional[bytes] = None
+        if isinstance(content_provider, (TextContentProvider, ByteContentProvider)):
+            # Feature to scan files which might be containers
+            data = content_provider.data
+            info = f"FILE:{content_provider.file_path}"
+        elif isinstance(content_provider, DiffContentProvider) and content_provider.diff:
+            candidates = self.scanner.scan(content_provider)
+            # Feature to scan binary diffs
+            diff = content_provider.diff[0].get("line")
+            # the check for legal fix mypy issue
+            if isinstance(diff, bytes):
+                data = diff
+            info = f"DIFF:{content_provider.file_path}"
+        else:
+            logger.warning(f"Content provider {type(content_provider)} does not support deep scan")
+            info = "NA"
+
+        if data:
+            data_provider = DataContentProvider(data=data,
+                                                file_path=content_provider.file_path,
+                                                file_type=content_provider.file_type,
+                                                info=content_provider.info or info)
+            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size - len(data))
+            augment_candidates(candidates, new_candidates)
+        return candidates

credsweeper-1.11.5/credsweeper/deep_scanner/deb_scanner.py

@@ -0,0 +1,55 @@
+import logging
+import struct
+from abc import ABC
+from typing import List, Optional, Generator, Tuple
+
+from credsweeper.common.constants import MIN_DATA_LEN, UTF_8
+from credsweeper.credentials import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.utils.util import Util
+
+logger = logging.getLogger(__name__)
+
+
+class DebScanner(AbstractScanner, ABC):
+    """Implements deb (ar) scanning"""
+
+    __header_size = 60
+
+    @staticmethod
+    def walk_deb(data: bytes) -> Generator[Tuple[int, str, bytes], None, None]:
+        """Processes sequence of DEB archive and yields offset, name and data"""
+        offset = 8  # b"!<arch>\n"
+        data_limit = len(data) - DebScanner.__header_size
+        while offset <= data_limit:
+            _data = data[offset:offset + DebScanner.__header_size]
+            offset += DebScanner.__header_size
+            # basic header structure
+            _name, _, _size, __ = struct.unpack('16s32s10s2s', _data)
+            file_size = int(_size)
+            if MIN_DATA_LEN < file_size <= len(data) - offset:
+                _data = data[offset:offset + file_size]
+                yield offset, _name.decode(encoding=UTF_8).strip().rstrip('/'), _data
+            offset += file_size if 0 == 1 & file_size else file_size + 1
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Extracts data file from .ar (debian) archive and launches data_scan"""
+        try:
+            candidates: List[Candidate] = []
+            for offset, name, data in DebScanner.walk_deb(data_provider.data):
+                deb_content_provider = DataContentProvider(data=data,
+                                                           file_path=f"{data_provider.file_path}/{name}",
+                                                           file_type=Util.get_extension(name),
+                                                           info=f"{data_provider.info}|DEB:0x{offset:x}")
+                new_limit = recursive_limit_size - len(data)
+                deb_candidates = self.recursive_scan(deb_content_provider, depth, new_limit)
+                candidates.extend(deb_candidates)
+            return candidates
+        except Exception as exc:
+            logger.error(exc)
+        return None