credsweeper 1.11.3__py3-none-any.whl → 1.11.5__py3-none-any.whl

credsweeper/deep_scanner/sqlite3_scanner.py

@@ -0,0 +1,79 @@
+import logging
+import os.path
+import sqlite3
+import sys
+import tempfile
+from abc import ABC
+from typing import List, Optional, Tuple, Any, Generator
+
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.struct_content_provider import StructContentProvider
+
+logger = logging.getLogger(__name__)
+
+
+class Sqlite3Scanner(AbstractScanner, ABC):
+    """Implements SQLite3 database scanning"""
+
+    @staticmethod
+    def __walk(sqlite3db) -> Generator[Tuple[str, Any], None, None]:
+        sqlite3db.row_factory = sqlite3.Row
+        cursor = sqlite3db.cursor()
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';")
+        for table in cursor.fetchall():
+            table_name = table[0]
+            try:
+                cursor.execute(f"SELECT * FROM {table_name}")
+                for row in cursor:
+                    yield table_name, dict(row)
+            except sqlite3.DatabaseError as exc:
+                print(f"Error reading table {table_name}: {exc}")
+
+    @staticmethod
+    def walk_sqlite(data: bytes) -> Generator[Tuple[str, Any], None, None]:
+        """Yields data from sqlite3 database"""
+        if 10 < sys.version_info.minor:
+            # Added in version 3.11
+            with sqlite3.connect(":memory:") as sqlite3db:
+                sqlite3db.deserialize(data)  # type: ignore
+                yield from Sqlite3Scanner.__walk(sqlite3db)
+        elif "nt" != os.name:
+            # a tmpfile has to be used. TODO: remove when 3.10 will deprecate
+            with tempfile.NamedTemporaryFile(suffix=".sqlite") as t:
+                t.write(data)
+                t.flush()
+                with sqlite3.connect(t.name) as sqlite3db:
+                    yield from Sqlite3Scanner.__walk(sqlite3db)
+        elif "nt" == os.name:
+            # windows trick. TODO: remove when 3.10 will deprecate
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".sqlite") as t:
+                t.write(data)
+                t.flush()
+            sqlite3db = sqlite3.connect(t.name)
+            yield from Sqlite3Scanner.__walk(sqlite3db)
+            sqlite3db.close()
+            if os.path.exists(t.name):
+                os.remove(t.name)
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Extracts data file from .ar (debian) archive and launches data_scan"""
+        try:
+            candidates: List[Candidate] = []
+            new_limit = recursive_limit_size - len(data_provider.data)
+            for table, row in self.walk_sqlite(data_provider.data):
+                struct_content_provider = StructContentProvider(struct=row,
+                                                                file_path=data_provider.file_path,
+                                                                file_type=data_provider.file_type,
+                                                                info=f"{data_provider.info}|SQLite3.{table}")
+                if new_candidates := self.structure_scan(struct_content_provider, depth, new_limit):
+                    candidates.extend(new_candidates)
+            return candidates
+        except Exception as exc:
+            logger.error(exc)
+        return None

credsweeper/file_handler/byte_content_provider.py

@@ -32,10 +32,10 @@ class ByteContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__data = None
-        if hasattr(self, "data"):
+        if "data" in self.__dict__:
             delattr(self, "data")
         self.__lines = None
-        if hasattr(self, "lines"):
+        if "lines" in self.__dict__:
             delattr(self, "lines")
 
     @cached_property

credsweeper/file_handler/content_provider.py

@@ -93,7 +93,7 @@ class ContentProvider(ABC):
             if min_len > len(line.strip()):
                 # Ignore target if stripped part is too short for all types
                 continue
-            elif MAX_LINE_LENGTH < len(line):
+            if MAX_LINE_LENGTH < len(line):
                 for chunk_start, chunk_end in Util.get_chunks(len(line)):
                     target = AnalysisTarget(
                         line_pos=line_pos,  #

credsweeper/file_handler/data_content_provider.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import string
 import warnings
 from functools import cached_property
 from typing import List, Optional, Any, Generator, Callable, Tuple
@@ -54,10 +53,10 @@ class DataContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__data = None
-        if hasattr(self, "data"):
+        if "data" in self.__dict__:
             delattr(self, "data")
         self.__text = None
-        if hasattr(self, "text"):
+        if "text" in self.__dict__:
             delattr(self, "text")
         self.structure = None
         self.decoded = None
@@ -385,7 +384,7 @@ class DataContentProvider(ContentProvider):
             return False
         try:
             self.decoded = Util.decode_base64(  #
-                self.text.translate(str.maketrans('', '', string.whitespace)),  #
+                text=Util.PEM_CLEANING_PATTERN.sub(r'', self.text).replace('\\', ''),  #
                 padding_safe=True,  #
                 urlsafe_detect=True)  #
         except Exception as exc:

credsweeper/file_handler/diff_content_provider.py

@@ -48,8 +48,8 @@ class DiffContentProvider(ContentProvider):
 
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
-        self.__diff = None
-        if hasattr(self, "diff"):
+        self.__diff = []
+        if "diff" in self.__dict__:
             delattr(self, "diff")
 
     @staticmethod

credsweeper/file_handler/file_path_extractor.py

@@ -162,7 +162,7 @@ class FilePathExtractor:
             True when the file is oversize or less than MIN_DATA_LEN, or unsupported
         """
         path = reference[1] if isinstance(reference, tuple) else reference
-        if isinstance(path, str) or isinstance(path, Path):
+        if isinstance(path, (str, Path)):
             file_size = os.path.getsize(path)
         elif isinstance(path, io.BytesIO):
             current_pos = path.tell()

credsweeper/file_handler/files_provider.py

@@ -42,7 +42,7 @@ class FilesProvider(AbstractProvider):
         """
         text_content_provider_list: List[Union[DiffContentProvider, TextContentProvider]] = []
         for path in self.paths:
-            if isinstance(path, str) or isinstance(path, Path):
+            if isinstance(path, (str, Path)):
                 new_files = FilePathExtractor.get_file_paths(config, path)
                 if self.skip_ignored:
                     new_files = FilePathExtractor.apply_gitignore(new_files)
@@ -50,9 +50,7 @@ class FilesProvider(AbstractProvider):
                     text_content_provider_list.append(TextContentProvider(_file))
             elif isinstance(path, io.BytesIO):
                 text_content_provider_list.append(TextContentProvider((":memory:", path)))
-            elif isinstance(path, tuple) \
-                    and (isinstance(path[0], str) or isinstance(path[0], Path)) \
-                    and isinstance(path[1], io.BytesIO):
+            elif isinstance(path, tuple) and (isinstance(path[0], (str, Path))) and isinstance(path[1], io.BytesIO):
                 # suppose, all the files must be scanned
                 text_content_provider_list.append(TextContentProvider(path))
             else:

credsweeper/file_handler/patches_provider.py

@@ -3,12 +3,12 @@ import logging
 from pathlib import Path
 from typing import List, Union, Tuple, Sequence
 
-from credsweeper import TextContentProvider
 from credsweeper.common.constants import DiffRowType
 from credsweeper.config import Config
 from credsweeper.file_handler.abstract_provider import AbstractProvider
 from credsweeper.file_handler.diff_content_provider import DiffContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
+from credsweeper.file_handler.text_content_provider import TextContentProvider
 from credsweeper.utils import Util
 
 logger = logging.getLogger(__name__)
@@ -37,11 +37,14 @@ class PatchesProvider(AbstractProvider):
         for file_path in self.paths:
             if FilePathExtractor.check_file_size(config, file_path):
                 continue
-            if isinstance(file_path, str) or isinstance(file_path, Path):
+            if isinstance(file_path, (str, Path)):
                 raw_patches.append(Util.read_file(file_path))
             elif isinstance(file_path, io.BytesIO):
                 the_patch = Util.decode_bytes(file_path.read())
                 raw_patches.append(the_patch)
+            elif isinstance(file_path, tuple) and 1 < len(file_path) and isinstance(file_path[1], io.BytesIO):
+                the_patch = Util.decode_bytes(file_path[1].read())
+                raw_patches.append(the_patch)
             else:
                 logger.error(f"Unknown path type: {file_path}")
 

credsweeper/file_handler/string_content_provider.py

@@ -38,10 +38,10 @@ class StringContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__lines = []
-        if hasattr(self, "lines"):
+        if "lines" in self.__dict__:
             delattr(self, "lines")
         self.__line_numbers = []
-        if hasattr(self, "line_numbers"):
+        if "line_numbers" in self.__dict__:
             delattr(self, "line_numbers")
 
     @cached_property

credsweeper/file_handler/struct_content_provider.py

@@ -38,7 +38,7 @@ class StructContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__struct = None
-        if hasattr(self, "struct"):
+        if "struct" in self.__dict__:
             delattr(self, "struct")
 
     def yield_analysis_target(self, min_len: int) -> Generator[AnalysisTarget, None, None]:

credsweeper/file_handler/text_content_provider.py

@@ -42,10 +42,10 @@ class TextContentProvider(ContentProvider):
     def free(self) -> None:
         """free data after scan to reduce memory usage"""
         self.__data = None
-        if hasattr(self, "data"):
+        if "data" in self.__dict__:
             delattr(self, "data")
         self.__lines = None
-        if hasattr(self, "lines"):
+        if "lines" in self.__dict__:
             delattr(self, "lines")
         if isinstance(self.__io, io.BytesIO) and self.__io and not self.__io.closed:
             self.__io.close()

credsweeper/filters/__init__.py

@@ -27,6 +27,7 @@ from credsweeper.filters.value_grafana_check import ValueGrafanaCheck
 from credsweeper.filters.value_grafana_service_check import ValueGrafanaServiceCheck
 from credsweeper.filters.value_hex_number_check import ValueHexNumberCheck
 from credsweeper.filters.value_jfrog_token_check import ValueJfrogTokenCheck
+from credsweeper.filters.value_json_web_key_check import ValueJsonWebKeyCheck
 from credsweeper.filters.value_json_web_token_check import ValueJsonWebTokenCheck
 from credsweeper.filters.value_last_word_check import ValueLastWordCheck
 from credsweeper.filters.value_method_check import ValueMethodCheck

credsweeper/filters/value_base64_encoded_pem_check.py

@@ -30,7 +30,7 @@ class ValueBase64EncodedPem(Filter):
         with contextlib.suppress(Exception):
             text = Util.decode_base64(line_data.value, padding_safe=True, urlsafe_detect=True)
             lines = text.decode(ASCII).splitlines()
-            lines_pos = [x for x in range(len(lines))]
+            lines_pos = list(range(len(lines)))
             for line_pos, line in zip(lines_pos, lines):
                 if PEM_BEGIN_PATTERN in line:
                     new_target = AnalysisTarget(line_pos, lines, lines_pos, target.descriptor)

credsweeper/filters/value_base64_key_check.py

@@ -1,7 +1,4 @@
 import contextlib
-import string
-
-from cryptography.hazmat.primitives import serialization
 
 from credsweeper.config import Config
 from credsweeper.credentials import LineData
@@ -13,6 +10,8 @@ from credsweeper.utils import Util
 class ValueBase64KeyCheck(Filter):
     """Check that candidate contains base64 encoded private key"""
 
+    EXTRA_TRANS_TABLE = str.maketrans('', '', "\",'\\")
+
     def __init__(self, config: Config = None) -> None:
         self.config = config
 
@@ -29,12 +28,10 @@ class ValueBase64KeyCheck(Filter):
         """
 
         with contextlib.suppress(Exception):
-            text = line_data.value
-            # replace to space any escaped sequence except space from string.whitespace
-            for x in ["\\t", "\\n", "\\r", "\\v", "\\f"]:
-                text = text.replace(x, ' ')
-            for x in string.whitespace:
-                text = text.replace(x, '')
+            # remove backslash escaping sequences
+            text = Util.PEM_CLEANING_PATTERN.sub(r'', line_data.value)
+            # remove whitespaces
+            text = text.translate(Util.WHITESPACE_TRANS_TABLE)
             # clean sequence concatenation case:
             text = text.replace("'+'", '')
             text = text.replace('"+"', '')
@@ -43,12 +40,10 @@ class ValueBase64KeyCheck(Filter):
             text = text.replace('%2F', '/')
             text = text.replace('%3D', '=')
             # clean any other chars which should not appear
-            for x in ["'", '"', '\\', ',']:
-                text = text.replace(x, "")
+            text = text.translate(ValueBase64KeyCheck.EXTRA_TRANS_TABLE)
             # only PEM standard encoding supported in regex pattern to cut off ending of the key
             key = Util.decode_base64(text, padding_safe=True, urlsafe_detect=False)
-            private_key = serialization.load_der_private_key(key, password=None)
-            if 0 < private_key.key_size:  # type: ignore
-                # access to size field check - some types have no size
+            private_key = Util.load_pk(key, password=None)
+            if Util.check_pk(private_key):
                 return False
         return True

credsweeper/filters/value_entropy_base64_check.py

@@ -19,12 +19,8 @@ class ValueEntropyBase64Check(ValueEntropyBaseCheck):
             y = 0.944 * math.log2(x) - 0.009 * x - 0.04
         elif 65 <= x < 256:
             y = 0.621 * math.log2(x) - 0.003 * x + 1.54
-        elif 256 <= x < 512:
-            y = 5.77
-        elif 512 <= x < 1024:
-            y = 5.89
-        elif 1024 <= x:
-            y = 5.94
+        elif 256 <= x:
+            y = 6 - 64 / x
         else:
             y = 0
         return y

credsweeper/filters/value_json_web_key_check.py

@@ -0,0 +1,37 @@
+import contextlib
+
+from credsweeper.config import Config
+from credsweeper.credentials import LineData
+from credsweeper.file_handler.analysis_target import AnalysisTarget
+from credsweeper.filters import Filter
+from credsweeper.utils import Util
+
+
+class ValueJsonWebKeyCheck(Filter):
+    """
+    Check that candidate is JWK which starts usually from 'e'
+    and have private parts of the key
+    https://datatracker.ietf.org/doc/html/rfc7517
+    https://datatracker.ietf.org/doc/html/rfc7518
+    """
+
+    def __init__(self, config: Config = None) -> None:
+        pass
+
+    def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
+        """Run filter checks on received key which might be structured.
+
+        Args:
+            line_data: credential candidate data
+            target: multiline target from which line data was obtained
+
+        Return:
+            True, when need to filter candidate and False if left
+
+        """
+        with contextlib.suppress(Exception):
+            if data := Util.decode_base64(line_data.value, padding_safe=True, urlsafe_detect=True):
+                if b'"kty":' in data and (b'"oct"' in data and b'"k":' in data or
+                                          (b'"EC"' in data or b'"RSA"' in data) and b'"d":' in data):
+                    return False
+        return True

credsweeper/filters/value_pattern_check.py

@@ -33,33 +33,33 @@ class ValuePatternCheck(Filter):
         # use non whitespace symbol pattern
         self.pattern = re.compile(fr"(\S)\1{{{str(self.pattern_len - 1)},}}")
 
-    def equal_pattern_check(self, line_data_value: str) -> bool:
+    def equal_pattern_check(self, value: str) -> bool:
         """Check if candidate value contain 4 and more same chars or numbers sequences.
 
         Args:
-            line_data_value: string variable, credential candidate value
+            value: string variable, credential candidate value
 
         Return:
             True if contain and False if not
 
         """
-        if self.pattern.findall(line_data_value):
+        if self.pattern.findall(value):
             return True
         return False
 
-    def ascending_pattern_check(self, line_data_value: str) -> bool:
+    def ascending_pattern_check(self, value: str) -> bool:
         """Check if candidate value contain 4 and more ascending chars or numbers sequences.
 
         Arg:
-            line_data_value: credential candidate value
+            value: credential candidate value
 
         Return:
             True if contain and False if not
 
         """
         count = 1
-        for key in range(len(line_data_value) - 1):
-            if ord(line_data_value[key + 1]) - ord(line_data_value[key]) == 1:
+        for key in range(len(value) - 1):
+            if ord(value[key + 1]) - ord(value[key]) == 1:
                 count += 1
             else:
                 count = 1
@@ -68,19 +68,19 @@ class ValuePatternCheck(Filter):
                 return True
         return False
 
-    def descending_pattern_check(self, line_data_value: str) -> bool:
+    def descending_pattern_check(self, value: str) -> bool:
         """Check if candidate value contain 4 and more descending chars or numbers sequences.
 
         Arg:
-            line_data_value: string variable, credential candidate value
+            value: string variable, credential candidate value
 
         Return:
             boolean variable. True if contain and False if not
 
         """
         count = 1
-        for key in range(len(line_data_value) - 1):
-            if ord(line_data_value[key]) - ord(line_data_value[key + 1]) == 1:
+        for key in range(len(value) - 1):
+            if ord(value[key]) - ord(value[key + 1]) == 1:
                 count += 1
             else:
                 count = 1
@@ -89,6 +89,57 @@ class ValuePatternCheck(Filter):
                 return True
         return False
 
+    def check_val(self, value: str) -> bool:
+        """Cumulative value check.
+
+        Arg:
+            value: string variable, credential candidate value
+
+        Return:
+            boolean variable. True if contain and False if not
+
+        """
+        if self.equal_pattern_check(value):
+            return True
+        if self.ascending_pattern_check(value):
+            return True
+        if self.descending_pattern_check(value):
+            return True
+        return False
+
+    def duple_pattern_check(self, value: str) -> bool:
+        """Check if candidate value is a duplet value with possible patterns.
+
+        Arg:
+            value: string variable, credential candidate value
+
+        Return:
+            boolean variable. True if contain and False if not
+
+        """
+        # 001122334455... case
+        pair_duple = True
+        # 0102030405... case
+        even_duple = True
+        even_prev = value[0]
+        even_value = value[0::2]
+        # 1020304050... case
+        odd_duple = True
+        odd_prev = value[1]
+        odd_value = value[1::2]
+        for even_i, odd_i in zip(even_value, odd_value):
+            pair_duple &= even_i == odd_i
+            even_duple &= even_i == even_prev
+            odd_duple &= odd_i == odd_prev
+            if not pair_duple and not even_duple and not odd_duple:
+                break
+        else:
+            if pair_duple or odd_duple:
+                return self.check_val(even_value)
+            if even_duple:
+                return self.check_val(odd_value)
+        return False
+
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
         """Run filter checks on received credential candidate data 'line_data'.
 
@@ -103,13 +154,10 @@ class ValuePatternCheck(Filter):
         if len(line_data.value) < self.pattern_len:
             return True
 
-        if self.equal_pattern_check(line_data.value):
-            return True
-
-        if self.ascending_pattern_check(line_data.value):
+        if self.check_val(line_data.value):
             return True
 
-        if self.descending_pattern_check(line_data.value):
+        if 2 * self.pattern_len <= len(line_data.value) and self.duple_pattern_check(line_data.value):
             return True
 
         return False

credsweeper/ml_model/features/file_extension.py

@@ -18,7 +18,7 @@ class FileExtension(WordIn):
         super().__init__(words=extensions)
 
     def __call__(self, candidates: List[Candidate]) -> np.ndarray:
-        extension_set = set([candidate.line_data_list[0].file_type.lower() for candidate in candidates])
+        extension_set = set(candidate.line_data_list[0].file_type.lower() for candidate in candidates)
         return self.word_in_set(extension_set)
 
     def extract(self, candidate: Candidate) -> Any: