credsweeper 1.11.4__py3-none-any.whl → 1.11.5__py3-none-any.whl

credsweeper/deep_scanner/deep_scanner.py

@@ -1,18 +1,8 @@
-import contextlib
-import datetime
 import logging
-from typing import List, Optional, Any, Tuple, Union
+from typing import List, Any, Tuple
 
-from credsweeper.common.constants import RECURSIVE_SCAN_LIMITATION, MIN_DATA_LEN, MIN_VALUE_LENGTH
+from credsweeper.common.constants import MIN_DATA_LEN
 from credsweeper.config import Config
-from credsweeper.credentials import Candidate
-from credsweeper.credentials.augment_candidates import augment_candidates
-from credsweeper.file_handler.byte_content_provider import ByteContentProvider
-from credsweeper.file_handler.content_provider import ContentProvider
-from credsweeper.file_handler.data_content_provider import DataContentProvider
-from credsweeper.file_handler.diff_content_provider import DiffContentProvider
-from credsweeper.file_handler.string_content_provider import StringContentProvider
-from credsweeper.file_handler.text_content_provider import TextContentProvider
 from credsweeper.scanner import Scanner
 from credsweeper.utils import Util
 from .byte_scanner import ByteScanner
@@ -23,21 +13,23 @@ from .eml_scanner import EmlScanner
 from .encoder_scanner import EncoderScanner
 from .gzip_scanner import GzipScanner
 from .html_scanner import HtmlScanner
+from .jclass_scanner import JclassScanner
 from .jks_scanner import JksScanner
 from .lang_scanner import LangScanner
 from .lzma_scanner import LzmaScanner
 from .mxfile_scanner import MxfileScanner
+from .patch_scanner import PatchScanner
 from .pdf_scanner import PdfScanner
-from .pkcs12_scanner import Pkcs12Scanner
+from .pkcs_scanner import PkcsScanner
 from .pptx_scanner import PptxScanner
+from .rpm_scanner import RpmScanner
+from .sqlite3_scanner import Sqlite3Scanner
 from .tar_scanner import TarScanner
 from .tmx_scanner import TmxScanner
 from .xlsx_scanner import XlsxScanner
 from .xml_scanner import XmlScanner
 from .zip_scanner import ZipScanner
-from ..common.constants import DEFAULT_ENCODING
-from ..file_handler.file_path_extractor import FilePathExtractor
-from ..file_handler.struct_content_provider import StructContentProvider
+from ..file_handler.descriptor import Descriptor
 
 logger = logging.getLogger(__name__)
 
@@ -49,12 +41,16 @@ class DeepScanner(
     EncoderScanner,  #
     GzipScanner,  #
     HtmlScanner,  #
+    JclassScanner,  #
     JksScanner,  #
     LangScanner,  #
     LzmaScanner,  #
+    PatchScanner,  #
     PdfScanner,  #
-    Pkcs12Scanner,  #
+    PkcsScanner,  #
     PptxScanner,  #
+    RpmScanner,  #
+    Sqlite3Scanner,  #
     TarScanner,  #
     DebScanner,  #
     XmlScanner,  #
@@ -82,7 +78,7 @@ class DeepScanner(
         return self.__scanner
 
     @staticmethod
-    def get_deep_scanners(data: bytes, file_type: str, depth: int) -> Tuple[List[Any], List[Any]]:
+    def get_deep_scanners(data: bytes, descriptor: Descriptor, depth: int) -> Tuple[List[Any], List[Any]]:
         """Returns possibly scan methods for the data depends on content and fallback scanners"""
         deep_scanners: List[Any] = []
         fallback_scanners: List[Any] = []
@@ -91,20 +87,20 @@ class DeepScanner(
                 deep_scanners.append(ZipScanner)
             # probably, there might be a docx, xlsx and so on.
             # It might be scanned with text representation in third-party libraries.
-            if file_type in (".xlsx", ".ods"):
+            if descriptor.extension in (".xlsx", ".ods"):
                 deep_scanners.append(XlsxScanner)
             else:
                 fallback_scanners.append(XlsxScanner)
-            if ".docx" == file_type:
+            if ".docx" == descriptor.extension:
                 deep_scanners.append(DocxScanner)
             else:
                 fallback_scanners.append(DocxScanner)
-            if ".pptx" == file_type:
+            if ".pptx" == descriptor.extension:
                 deep_scanners.append(PptxScanner)
             else:
                 fallback_scanners.append(PptxScanner)
         elif Util.is_com(data):
-            if ".xls" == file_type:
+            if ".xls" == descriptor.extension:
                 deep_scanners.append(XlsxScanner)
             else:
                 fallback_scanners.append(XlsxScanner)
@@ -125,10 +121,18 @@ class DeepScanner(
                 deep_scanners.append(GzipScanner)
         elif Util.is_pdf(data):
             deep_scanners.append(PdfScanner)
+        elif Util.is_rpm(data):
+            if 0 < depth:
+                deep_scanners.append(RpmScanner)
+        elif Util.is_jclass(data):
+            deep_scanners.append(JclassScanner)
         elif Util.is_jks(data):
             deep_scanners.append(JksScanner)
+        elif Util.is_sqlite3(data):
+            if 0 < depth:
+                deep_scanners.append(Sqlite3Scanner)
         elif Util.is_asn1(data):
-            deep_scanners.append(Pkcs12Scanner)
+            deep_scanners.append(PkcsScanner)
         elif Util.is_xml(data):
             if Util.is_html(data):
                 deep_scanners.append(HtmlScanner)
@@ -146,9 +150,12 @@ class DeepScanner(
                 deep_scanners.append(XmlScanner)
                 fallback_scanners.append(ByteScanner)
         elif Util.is_eml(data):
-            if ".eml" == file_type:
+            if ".eml" == descriptor.extension:
                 deep_scanners.append(EmlScanner)
             else:
+                if 0 < depth:
+                    # formal patch looks like an eml
+                    deep_scanners.append(PatchScanner)
                 fallback_scanners.append(EmlScanner)
             fallback_scanners.append(ByteScanner)
         elif Util.is_known(data):
@@ -156,231 +163,11 @@ class DeepScanner(
             pass
         elif not Util.is_binary(data):
             if 0 < depth:
+                deep_scanners.append(PatchScanner)
                 deep_scanners.append(EncoderScanner)
                 deep_scanners.append(LangScanner)
             deep_scanners.append(ByteScanner)
         else:
-            logger.warning("Cannot apply a deep scanner for type %s prefix %s", file_type, str(data[:MIN_DATA_LEN]))
+            logger.warning("Cannot apply a deep scanner for type %s prefix %s %d", descriptor,
+                           repr(data[:MIN_DATA_LEN]), len(data))
         return deep_scanners, fallback_scanners
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
-    def deep_scan_with_fallback(self, data_provider: DataContentProvider, depth: int,
-                                recursive_limit_size: int) -> List[Candidate]:
-        """Scans with deep scanners and fallback scanners if possible
-
-            Args:
-                data_provider: DataContentProvider with raw data
-                depth: maximal level of recursion
-                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
-
-            Returns: list with candidates
-
-        """
-        candidates: List[Candidate] = []
-        deep_scanners, fallback_scanners = self.get_deep_scanners(data_provider.data, data_provider.file_type, depth)
-        fallback = True
-        for scan_class in deep_scanners:
-            new_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
-            if new_candidates is None:
-                # scanner did not recognise the content type
-                continue
-            augment_candidates(candidates, new_candidates)
-            # this scan is successful, so fallback is not necessary
-            fallback = False
-        if fallback:
-            for scan_class in fallback_scanners:
-                fallback_candidates = scan_class.data_scan(self, data_provider, depth, recursive_limit_size)
-                if fallback_candidates is None:
-                    continue
-                augment_candidates(candidates, fallback_candidates)
-                # use only first successful fallback scanner
-                break
-        return candidates
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
-    def scan(self,
-             content_provider: ContentProvider,
-             depth: int,
-             recursive_limit_size: Optional[int] = None) -> List[Candidate]:
-        """Initial scan method to launch recursive scan. Skips ByteScanner to prevent extra scan
-
-            Args:
-                content_provider: ContentProvider that might contain raw data
-                depth: maximal level of recursion
-                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
-        """
-        recursive_limit_size = recursive_limit_size if isinstance(recursive_limit_size,
-                                                                  int) else RECURSIVE_SCAN_LIMITATION
-        candidates: List[Candidate] = []
-        data: Optional[bytes] = None
-        if isinstance(content_provider, (TextContentProvider, ByteContentProvider)):
-            # Feature to scan files which might be containers
-            data = content_provider.data
-            info = f"FILE:{content_provider.file_path}"
-        elif isinstance(content_provider, DiffContentProvider) and content_provider.diff:
-            candidates = self.scanner.scan(content_provider)
-            # Feature to scan binary diffs
-            diff = content_provider.diff[0].get("line")
-            # the check for legal fix mypy issue
-            if isinstance(diff, bytes):
-                data = diff
-            info = f"DIFF:{content_provider.file_path}"
-        else:
-            logger.warning(f"Content provider {type(content_provider)} does not support deep scan")
-            info = "NA"
-
-        if data:
-            data_provider = DataContentProvider(data=data,
-                                                file_path=content_provider.file_path,
-                                                file_type=content_provider.file_type,
-                                                info=content_provider.info or info)
-            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size - len(data))
-            augment_candidates(candidates, new_candidates)
-        return candidates
-
-    def recursive_scan(
-            self,  #
-            data_provider: DataContentProvider,  #
-            depth: int = 0,  #
-            recursive_limit_size: int = 0) -> List[Candidate]:
-        """Recursive function to scan files which might be containers like ZIP archives
-
-            Args:
-                data_provider: DataContentProvider object may be a container
-                depth: maximal level of recursion
-                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
-        """
-        candidates: List[Candidate] = []
-        if 0 > depth:
-            # break recursion if maximal depth is reached
-            logger.debug("Bottom reached %s recursive_limit_size:%d", data_provider.file_path, recursive_limit_size)
-            return candidates
-        depth -= 1
-        if MIN_DATA_LEN > len(data_provider.data):
-            # break recursion for minimal data size
-            logger.debug("Too small data: size=%d, depth=%d, limit=%d, path=%s, info=%s", len(data_provider.data),
-                         depth, recursive_limit_size, data_provider.file_path, data_provider.info)
-            return candidates
-        logger.debug("Start data_scan: size=%d, depth=%d, limit=%d, path=%s, info=%s", len(data_provider.data), depth,
-                     recursive_limit_size, data_provider.file_path, data_provider.info)
-
-        if FilePathExtractor.is_find_by_ext_file(self.config, data_provider.file_type):
-            # Skip scanning file and makes fake candidate due the extension is suspicious
-            dummy_candidate = Candidate.get_dummy_candidate(self.config, data_provider.file_path,
-                                                            data_provider.file_type, data_provider.info,
-                                                            FilePathExtractor.FIND_BY_EXT_RULE)
-            candidates.append(dummy_candidate)
-        else:
-            new_candidates = self.deep_scan_with_fallback(data_provider, depth, recursive_limit_size)
-            augment_candidates(candidates, new_candidates)
-
-        return candidates
-
-    def structure_scan(
-            self,  #
-            struct_provider: StructContentProvider,  #
-            depth: int,  #
-            recursive_limit_size: int) -> List[Candidate]:
-        """Recursive function to scan structured data
-
-            Args:
-                struct_provider: DataContentProvider object may be a container
-                depth: maximal level of recursion
-                recursive_limit_size: maximal bytes of opened files to prevent recursive zip-bomb attack
-        """
-        candidates: List[Candidate] = []
-        logger.debug("Start struct_scan: depth=%d, limit=%d, path=%s, info=%s", depth, recursive_limit_size,
-                     struct_provider.file_path, struct_provider.info)
-
-        if 0 > depth:
-            # break recursion if maximal depth is reached
-            logger.debug("bottom reached %s recursive_limit_size:%d", struct_provider.file_path, recursive_limit_size)
-            return candidates
-
-        depth -= 1
-
-        items: List[Tuple[Union[int, str], Any]] = []
-        struct_key: Optional[str] = None
-        struct_value: Optional[str] = None
-        lines_for_keyword_rules = []
-        if isinstance(struct_provider.struct, dict):
-            for key, value in struct_provider.struct.items():
-                if isinstance(value, (list, tuple)) and 1 == len(value):
-                    # simplify some structures like YAML when single item in new line is a value
-                    items.append((key, value[0]))
-                else:
-                    items.append((key, value))
-            # for transformation {"key": "api_key", "value": "XXXXXXX"} -> {"api_key": "XXXXXXX"}
-            struct_key = struct_provider.struct.get("key")
-            struct_value = struct_provider.struct.get("value")
-        elif isinstance(struct_provider.struct, (list, tuple)):
-            items = list(enumerate(struct_provider.struct))
-        else:
-            logger.error("Not supported type:%s val:%s", str(type(struct_provider.struct)), str(struct_provider.struct))
-
-        for key, value in items:
-            if isinstance(value, dict) or isinstance(value, (list, tuple)) and 1 <= len(value):
-                val_struct_provider = StructContentProvider(struct=value,
-                                                            file_path=struct_provider.file_path,
-                                                            file_type=struct_provider.file_type,
-                                                            info=f"{struct_provider.info}|STRUCT:{key}")
-                new_candidates = self.structure_scan(val_struct_provider, depth, recursive_limit_size)
-                candidates.extend(new_candidates)
-
-            elif isinstance(value, bytes):
-                if MIN_DATA_LEN <= len(value):
-                    bytes_struct_provider = DataContentProvider(data=value,
-                                                                file_path=struct_provider.file_path,
-                                                                file_type=struct_provider.file_type,
-                                                                info=f"{struct_provider.info}|BYTES:{key}")
-                    new_limit = recursive_limit_size - len(value)
-                    new_candidates = self.recursive_scan(bytes_struct_provider, depth, new_limit)
-                    candidates.extend(new_candidates)
-                if MIN_VALUE_LENGTH <= len(value) and isinstance(key, str) \
-                        and self.scanner.keywords_required_substrings_check(key.lower()):
-                    str_val = str(value)
-                    lines_for_keyword_rules.append(f"{key} = '{str_val}'" if '"' in str_val else f'{key} = "{str_val}"')
-
-            elif isinstance(value, str):
-                if MIN_DATA_LEN <= len(value):
-                    # recursive scan only for data which may be decoded at least
-                    with contextlib.suppress(UnicodeError):
-                        data = value.encode(encoding=DEFAULT_ENCODING, errors='strict')
-                        str_struct_provider = DataContentProvider(data=data,
-                                                                  file_path=struct_provider.file_path,
-                                                                  file_type=struct_provider.file_type,
-                                                                  info=f"{struct_provider.info}|STRING:{key}")
-                        new_limit = recursive_limit_size - len(str_struct_provider.data)
-                        new_candidates = self.recursive_scan(str_struct_provider, depth, new_limit)
-                        candidates.extend(new_candidates)
-                # use key = "value" scan for common cases like in TOML
-                if MIN_VALUE_LENGTH <= len(value) and isinstance(key, str) \
-                        and self.scanner.keywords_required_substrings_check(key.lower()):
-                    lines_for_keyword_rules.append(f"{key} = '{value}'" if '"' in value else f'{key} = "{value}"')
-
-            elif isinstance(value, (int, float, datetime.date, datetime.datetime)):
-                # skip useless types
-                pass
-            else:
-                logger.warning("Not supported type:%s value(%s)", str(type(value)), str(value))
-
-        if lines_for_keyword_rules:
-            str_provider = StringContentProvider(lines_for_keyword_rules,
-                                                 file_path=struct_provider.file_path,
-                                                 file_type=".py",
-                                                 info=f"{struct_provider.info}|KEYWORD:`{lines_for_keyword_rules}`")
-            new_candidates = self.scanner.scan(str_provider)
-            augment_candidates(candidates, new_candidates)
-
-        # last check when dictionary is {"key": "api_key", "value": "XXXXXXX"} -> {"api_key": "XXXXXXX"}
-        if isinstance(struct_key, str) and isinstance(struct_value, str):
-            key_value_provider = StringContentProvider(
-                [f"{struct_key} = '{struct_value}'" if '"' in struct_value else f'{struct_key} = "{struct_value}"'],
-                file_path=struct_provider.file_path,
-                file_type=".toml",
-                info=f"{struct_provider.info}|KEY_VALUE:`{lines_for_keyword_rules}`")
-            new_candidates = self.scanner.scan(key_value_provider)
-            augment_candidates(candidates, new_candidates)
-        return candidates

credsweeper/deep_scanner/jclass_scanner.py

@@ -0,0 +1,74 @@
+import io
+import logging
+import struct
+from abc import ABC
+from typing import List, Optional
+
+from credsweeper.common.constants import MIN_DATA_LEN, UTF_8
+from credsweeper.credentials import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.struct_content_provider import StructContentProvider
+
+logger = logging.getLogger(__name__)
+
+
+class JclassScanner(AbstractScanner, ABC):
+    """Implements java .class scanning"""
+
+    @staticmethod
+    def u2(stream: io.BytesIO) -> int:
+        """Extracts unsigned 16 bit big-endian"""
+        return int(struct.unpack(">H", stream.read(2))[0])
+
+    @staticmethod
+    def get_utf8_constants(stream: io.BytesIO) -> List[str]:
+        """Extracts only Utf8 constants from java ClassFile"""
+        result = []
+        item_count = JclassScanner.u2(stream)
+        while 0 < item_count:
+            # actual number of items is one less!
+            item_count -= 1
+            # uint8
+            tag = int(stream.read(1)[0])
+            if 1 == tag:
+                length = JclassScanner.u2(stream)
+                data = stream.read(int(length))
+                if MIN_DATA_LEN <= length:
+                    value = data.decode(encoding=UTF_8, errors="replace")
+                    result.append(value)
+            elif tag in (3, 4, 9, 10, 11, 12, 18):
+                _ = stream.read(4)
+            elif tag in (7, 8, 16):
+                _ = stream.read(2)
+            elif tag in (5, 6):
+                _ = stream.read(8)
+            elif 15 == tag:
+                _ = stream.read(3)
+            else:
+                logger.error(f"Unknown tag {tag}")
+                break
+        return result
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Extracts data from binary"""
+        try:
+            stream = io.BytesIO(data_provider.data)
+            stream.read(4)  # magic
+            minor = JclassScanner.u2(stream)
+            major = JclassScanner.u2(stream)
+            constants = JclassScanner.get_utf8_constants(stream)
+            struct_content_provider = StructContentProvider(struct=constants,
+                                                            file_path=data_provider.file_path,
+                                                            file_type=data_provider.file_type,
+                                                            info=f"{data_provider.info}|Java.{major}.{minor}")
+            new_limit = recursive_limit_size - sum(len(x) for x in constants)
+            candidates = self.structure_scan(struct_content_provider, depth, new_limit)
+            return candidates
+        except Exception as jclass_exc:
+            logger.error(f"{data_provider.file_path}:{jclass_exc}")
+        return None

credsweeper/deep_scanner/patch_scanner.py

@@ -0,0 +1,48 @@
+import io
+import logging
+from abc import ABC
+from typing import List, Optional
+
+from credsweeper.common.constants import DiffRowType
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.patches_provider import PatchesProvider
+
+logger = logging.getLogger(__name__)
+
+
+class PatchScanner(AbstractScanner, ABC):
+    """Implements .patch scanning"""
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Tries to scan EML with text representation"""
+        try:
+            candidates: List[Candidate] = []
+            # common limitation
+            new_limit_size = recursive_limit_size - len(data_provider.data)
+            # ADDED
+            path_added = [(data_provider.file_path, io.BytesIO(data_provider.data))]
+            added_content_provider = PatchesProvider(path_added, change_type=DiffRowType.ADDED)
+            for added_file in added_content_provider.get_scannable_files(self.config):
+                added_candidates = self.scan(added_file, depth, new_limit_size)
+                candidates.extend(added_candidates)
+            # DELETED
+            path_deleted = [(data_provider.file_path, io.BytesIO(data_provider.data))]
+            deleted_content_provider = PatchesProvider(path_deleted, change_type=DiffRowType.DELETED)
+            for deleted_file in deleted_content_provider.get_scannable_files(self.config):
+                added_candidates = self.scan(deleted_file, depth, new_limit_size)
+                candidates.extend(added_candidates)
+            # update the line data for deep scan only
+            for i in candidates:
+                for line_data in i.line_data_list:
+                    line_data.path = f"{data_provider.file_path}/{line_data.path}"
+                    line_data.info = f"{data_provider.info}|PATCH:{line_data.info}"
+            return candidates
+        except Exception as patch_exc:
+            logger.error(f"{data_provider.file_path}:{patch_exc}")
+        return None

credsweeper/deep_scanner/pkcs_scanner.py

@@ -0,0 +1,41 @@
+import base64
+import logging
+from abc import ABC
+from typing import List, Optional
+
+from credsweeper.credentials import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.utils import Util
+
+logger = logging.getLogger(__name__)
+
+
+class PkcsScanner(AbstractScanner, ABC):
+    """Implements pkcs12 scanning"""
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Tries to scan PKCS12 to open with standard password"""
+        for pw_probe in self.config.bruteforce_list:
+            try:
+                password = pw_probe.encode() if pw_probe else None
+                if pkey := Util.load_pk(data_provider.data, password):
+                    if not Util.check_pk(pkey):
+                        logger.debug("False alarm %s", data_provider.info)
+                        return []
+                    candidate = Candidate.get_dummy_candidate(
+                        self.config,  #
+                        data_provider.file_path,  #
+                        data_provider.file_type,  #
+                        f"{data_provider.info}|PKCS:{repr(password)} is the password",  #
+                        "PKCS")
+                    candidate.line_data_list[0].line = base64.b64encode(data_provider.data).decode()
+                    candidate.line_data_list[0].value = repr(password)
+                    return [candidate]
+            except Exception as pkcs_exc:
+                logger.debug(f"{data_provider.file_path}:{pw_probe}:{pkcs_exc}")
+        return None

credsweeper/deep_scanner/rpm_scanner.py

@@ -0,0 +1,49 @@
+import io
+import logging
+from abc import ABC
+from typing import List, Optional
+
+import rpmfile
+
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.file_path_extractor import FilePathExtractor
+from credsweeper.utils import Util
+
+logger = logging.getLogger(__name__)
+
+
+class RpmScanner(AbstractScanner, ABC):
+    """Implements rpm scanning"""
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Extracts files one by one from the package type and launches recursive scan"""
+        try:
+            candidates = []
+            with rpmfile.open(fileobj=io.BytesIO(data_provider.data)) as rpm_file:
+                for member in rpm_file.getmembers():
+                    # skip directory
+                    if 0 != member.isdir:
+                        continue
+                    if FilePathExtractor.check_exclude_file(self.config, member.name):
+                        continue
+                    if 0 > recursive_limit_size - member.size:
+                        logger.error(f"{member.filename}: size {member.size}"
+                                     f" is over limit {recursive_limit_size} depth:{depth}")
+                        continue
+                    rpm_content_provider = DataContentProvider(data=rpm_file.extractfile(member).read(),
+                                                               file_path=data_provider.file_path,
+                                                               file_type=Util.get_extension(member.name),
+                                                               info=f"{data_provider.info}|RPM:{member.name}")
+                    new_limit = recursive_limit_size - len(rpm_content_provider.data)
+                    rpm_candidates = self.recursive_scan(rpm_content_provider, depth, new_limit)
+                    candidates.extend(rpm_candidates)
+            return candidates
+        except Exception as rpm_exc:
+            logger.error(f"{data_provider.file_path}:{rpm_exc}")
+        return None

credsweeper/deep_scanner/sqlite3_scanner.py

@@ -0,0 +1,79 @@
+import logging
+import os.path
+import sqlite3
+import sys
+import tempfile
+from abc import ABC
+from typing import List, Optional, Tuple, Any, Generator
+
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.struct_content_provider import StructContentProvider
+
+logger = logging.getLogger(__name__)
+
+
+class Sqlite3Scanner(AbstractScanner, ABC):
+    """Implements SQLite3 database scanning"""
+
+    @staticmethod
+    def __walk(sqlite3db) -> Generator[Tuple[str, Any], None, None]:
+        sqlite3db.row_factory = sqlite3.Row
+        cursor = sqlite3db.cursor()
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';")
+        for table in cursor.fetchall():
+            table_name = table[0]
+            try:
+                cursor.execute(f"SELECT * FROM {table_name}")
+                for row in cursor:
+                    yield table_name, dict(row)
+            except sqlite3.DatabaseError as exc:
+                print(f"Error reading table {table_name}: {exc}")
+
+    @staticmethod
+    def walk_sqlite(data: bytes) -> Generator[Tuple[str, Any], None, None]:
+        """Yields data from sqlite3 database"""
+        if 10 < sys.version_info.minor:
+            # Added in version 3.11
+            with sqlite3.connect(":memory:") as sqlite3db:
+                sqlite3db.deserialize(data)  # type: ignore
+                yield from Sqlite3Scanner.__walk(sqlite3db)
+        elif "nt" != os.name:
+            # a tmpfile has to be used. TODO: remove when 3.10 will deprecate
+            with tempfile.NamedTemporaryFile(suffix=".sqlite") as t:
+                t.write(data)
+                t.flush()
+                with sqlite3.connect(t.name) as sqlite3db:
+                    yield from Sqlite3Scanner.__walk(sqlite3db)
+        elif "nt" == os.name:
+            # windows trick. TODO: remove when 3.10 will deprecate
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".sqlite") as t:
+                t.write(data)
+                t.flush()
+            sqlite3db = sqlite3.connect(t.name)
+            yield from Sqlite3Scanner.__walk(sqlite3db)
+            sqlite3db.close()
+            if os.path.exists(t.name):
+                os.remove(t.name)
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Extracts data file from .ar (debian) archive and launches data_scan"""
+        try:
+            candidates: List[Candidate] = []
+            new_limit = recursive_limit_size - len(data_provider.data)
+            for table, row in self.walk_sqlite(data_provider.data):
+                struct_content_provider = StructContentProvider(struct=row,
+                                                                file_path=data_provider.file_path,
+                                                                file_type=data_provider.file_type,
+                                                                info=f"{data_provider.info}|SQLite3.{table}")
+                if new_candidates := self.structure_scan(struct_content_provider, depth, new_limit):
+                    candidates.extend(new_candidates)
+            return candidates
+        except Exception as exc:
+            logger.error(exc)
+        return None