credsweeper 1.11.5__py3-none-any.whl → 1.12.0__py3-none-any.whl

credsweeper/deep_scanner/html_scanner.py

@@ -2,7 +2,7 @@ import logging
 from abc import ABC
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.string_content_provider import StringContentProvider

credsweeper/deep_scanner/jclass_scanner.py

@@ -5,7 +5,7 @@ from abc import ABC
 from typing import List, Optional
 
 from credsweeper.common.constants import MIN_DATA_LEN, UTF_8
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.struct_content_provider import StructContentProvider

credsweeper/deep_scanner/jks_scanner.py

@@ -4,7 +4,7 @@ from typing import List, Optional
 
 import jks
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 

credsweeper/deep_scanner/lang_scanner.py

@@ -2,7 +2,7 @@ import logging
 from abc import ABC
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.struct_content_provider import StructContentProvider

credsweeper/deep_scanner/lzma_scanner.py

@@ -4,10 +4,10 @@ from abc import ABC
 from pathlib import Path
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/deep_scanner/mxfile_scanner.py

@@ -5,7 +5,7 @@ from typing import List, Optional
 from bs4 import BeautifulSoup
 from lxml import etree
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.string_content_provider import StringContentProvider

credsweeper/deep_scanner/pdf_scanner.py

@@ -6,7 +6,7 @@ from typing import List, Optional
 from pdfminer.high_level import extract_pages
 from pdfminer.layout import LAParams, LTText, LTItem
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider, MIN_DATA_LEN
 from credsweeper.file_handler.string_content_provider import StringContentProvider

credsweeper/deep_scanner/pkcs_scanner.py

@@ -3,10 +3,10 @@ import logging
 from abc import ABC
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/deep_scanner/pptx_scanner.py

@@ -5,7 +5,7 @@ from typing import List, Optional
 
 from pptx import Presentation
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.string_content_provider import StringContentProvider

credsweeper/deep_scanner/rpm_scanner.py

@@ -9,7 +9,7 @@ from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/deep_scanner/tar_scanner.py

@@ -4,11 +4,11 @@ from abc import ABC
 from tarfile import TarFile
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/deep_scanner/tmx_scanner.py

@@ -5,11 +5,11 @@ from typing import List, Optional
 from lxml import etree
 
 from credsweeper.common.constants import MIN_DATA_LEN
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.string_content_provider import StringContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/deep_scanner/xlsx_scanner.py

@@ -5,12 +5,12 @@ from typing import List, Optional
 
 import pandas as pd
 
-from credsweeper.credentials import Candidate
 from credsweeper.credentials.augment_candidates import augment_candidates
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.string_content_provider import StringContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/deep_scanner/xml_scanner.py

@@ -2,7 +2,7 @@ import logging
 from abc import ABC
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.string_content_provider import StringContentProvider

credsweeper/deep_scanner/zip_scanner.py

@@ -4,11 +4,11 @@ from abc import ABC
 from typing import List, Optional
 from zipfile import ZipFile
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/file_handler/__init__.py

@@ -1,15 +0,0 @@
-from credsweeper.file_handler.byte_content_provider import ByteContentProvider
-from credsweeper.file_handler.content_provider import ContentProvider
-from credsweeper.file_handler.data_content_provider import DataContentProvider
-from credsweeper.file_handler.diff_content_provider import DiffContentProvider
-from credsweeper.file_handler.string_content_provider import StringContentProvider
-from credsweeper.file_handler.text_content_provider import TextContentProvider
-
-__all__ = [
-    'ByteContentProvider',  #
-    'ContentProvider',  #
-    'DataContentProvider',  #
-    'DiffContentProvider',  #
-    'StringContentProvider',  #
-    'TextContentProvider',  #
-]

credsweeper/file_handler/abstract_provider.py

@@ -3,9 +3,8 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Union, Tuple, Sequence
 
-from credsweeper.config import Config
-from credsweeper.file_handler.diff_content_provider import DiffContentProvider
-from credsweeper.file_handler.text_content_provider import TextContentProvider
+from credsweeper.config.config import Config
+from credsweeper.file_handler.content_provider import ContentProvider
 
 
 class AbstractProvider(ABC):
@@ -31,7 +30,7 @@ class AbstractProvider(ABC):
         self.__paths = paths
 
     @abstractmethod
-    def get_scannable_files(self, config: Config) -> Sequence[Union[DiffContentProvider, TextContentProvider]]:
+    def get_scannable_files(self, config: Config) -> Sequence[ContentProvider]:
         """Get list of file object for analysis based on attribute "paths".
 
         Args:

credsweeper/file_handler/byte_content_provider.py

@@ -3,7 +3,7 @@ from typing import List, Optional, Generator
 
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.file_handler.content_provider import ContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 
 class ByteContentProvider(ContentProvider):

credsweeper/file_handler/content_provider.py

@@ -6,7 +6,7 @@ from typing import List, Optional, Generator
 from credsweeper.common.constants import MAX_LINE_LENGTH
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.file_handler.descriptor import Descriptor
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/file_handler/data_content_provider.py

@@ -10,7 +10,7 @@ from bs4 import BeautifulSoup, Tag, XMLParsedAsHTMLWarning
 from credsweeper.common.constants import MIN_DATA_LEN
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.file_handler.content_provider import ContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning, module='bs4')
 logger = logging.getLogger(__name__)

credsweeper/file_handler/diff_content_provider.py

@@ -1,14 +1,34 @@
 import logging
+from dataclasses import dataclass
 from functools import cached_property
-from typing import List, Tuple, Generator
+from typing import List, Tuple, Generator, TypedDict, Optional, Union, Any, Dict
+
+import whatthepatch
 
 from credsweeper.common.constants import DiffRowType
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.file_handler.content_provider import ContentProvider
-from credsweeper.utils import DiffRowData, Util, DiffDict
 
 logger = logging.getLogger(__name__)
 
+DiffDict = TypedDict(
+    "DiffDict",
+    {
+        "old": Optional[int],  #
+        "new": Optional[int],  #
+        "line": Union[str, bytes],  # bytes are possibly since whatthepatch v1.0.4
+        "hunk": Any  # not used
+    })
+
+
+@dataclass(frozen=True)
+class DiffRowData:
+    """Class for keeping data of diff row."""
+
+    line_type: DiffRowType
+    line_numb: int
+    line: str
+
 
 class DiffContentProvider(ContentProvider):
     """Provide data from a single `.patch` file.
@@ -76,6 +96,116 @@ class DiffContentProvider(ContentProvider):
                 all_lines.append(line_data.line)
         return change_numbs, all_lines
 
+    @staticmethod
+    def patch2files_diff(raw_patch: List[str], change_type: DiffRowType) -> Dict[str, List[DiffDict]]:
+        """Generate files changes from patch for added or deleted filepaths.
+
+        Args:
+            raw_patch: git patch file content
+            change_type: change type to select, DiffRowType.ADDED or DiffRowType.DELETED
+
+        Return:
+            return dict with ``{file paths: list of file row changes}``, where
+            elements of list of file row changes represented as::
+
+                {
+                    "old": line number before diff,
+                    "new": line number after diff,
+                    "line": line text,
+                    "hunk": diff hunk number
+                }
+
+        """
+        if not raw_patch:
+            return {}
+
+        added_files, deleted_files = {}, {}
+        try:
+            for patch in whatthepatch.parse_patch(raw_patch):
+                if patch.changes is None:
+                    logger.warning(f"Patch '{str(patch.header)}' cannot be scanned")
+                    continue
+                changes = []
+                for change in patch.changes:
+                    change_dict = change._asdict()
+                    changes.append(change_dict)
+
+                added_files[patch.header.new_path] = changes
+                deleted_files[patch.header.old_path] = changes
+            if change_type == DiffRowType.ADDED:
+                return added_files
+            elif change_type == DiffRowType.DELETED:
+                return deleted_files
+            else:
+                logger.error(f"Change type should be one of: '{DiffRowType.ADDED}', '{DiffRowType.DELETED}';"
+                             f" but received {change_type}")
+        except Exception as exc:
+            logger.exception(exc)
+        return {}
+
+    @staticmethod
+    def preprocess_diff_rows(
+            added_line_number: Optional[int],  #
+            deleted_line_number: Optional[int],  #
+            line: str) -> List[DiffRowData]:
+        """Auxiliary function to extend diff changes.
+
+        Args:
+            added_line_number: number of added line or None
+            deleted_line_number: number of deleted line or None
+            line: the text line
+
+        Return:
+            diff rows data with as list of row change type, line number, row content
+
+        """
+        rows_data: List[DiffRowData] = []
+        if isinstance(added_line_number, int):
+            # indicates line was inserted
+            rows_data.append(DiffRowData(DiffRowType.ADDED, added_line_number, line))
+        if isinstance(deleted_line_number, int):
+            # indicates line was removed
+            rows_data.append(DiffRowData(DiffRowType.DELETED, deleted_line_number, line))
+        return rows_data
+
+    @staticmethod
+    def wrong_change(change: DiffDict) -> bool:
+        """Returns True if the change is wrong"""
+        for i in ["line", "new", "old"]:
+            if i not in change:
+                logger.error(f"Skipping wrong change {change}")
+                return True
+        return False
+
+    @staticmethod
+    def preprocess_file_diff(changes: List[DiffDict]) -> List[DiffRowData]:
+        """Generate changed file rows from diff data with changed lines (e.g. marked + or - in diff).
+
+        Args:
+            changes: git diff by file rows data
+
+        Return:
+            diff rows data with as list of row change type, line number, row content
+
+        """
+        if not changes:
+            return []
+
+        rows_data = []
+        # process diff to restore lines and their positions
+        for change in changes:
+            if DiffContentProvider.wrong_change(change):
+                continue
+            line = change["line"]
+            if isinstance(line, str):
+                rows_data.extend(DiffContentProvider.preprocess_diff_rows(change.get("new"), change.get("old"), line))
+            elif isinstance(line, (bytes, bytearray)):
+                logger.warning("The feature is available with the deep scan option")
+            else:
+                logger.error(f"Unknown type of line {type(line)}")
+
+        return rows_data
+
     def yield_analysis_target(self, min_len: int) -> Generator[AnalysisTarget, None, None]:
         """Preprocess file diff data to scan.
 
@@ -86,6 +216,6 @@ class DiffContentProvider(ContentProvider):
             list of analysis targets of every row of file diff corresponding to change type "self.change_type"
 
         """
-        lines_data = Util.preprocess_file_diff(self.__diff)
+        lines_data = DiffContentProvider.preprocess_file_diff(self.__diff)
         change_numbs, all_lines = self.parse_lines_data(self.__change_type, lines_data)
         return self.lines_to_targets(min_len, all_lines, change_numbs)

credsweeper/file_handler/file_path_extractor.py

@@ -7,8 +7,8 @@ from typing import List, Dict, Union, Tuple
 from git import InvalidGitRepositoryError, NoSuchPathError, Repo
 
 from credsweeper.common.constants import MIN_DATA_LEN
-from credsweeper.config import Config
-from credsweeper.utils import Util
+from credsweeper.config.config import Config
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/file_handler/files_provider.py

@@ -3,9 +3,9 @@ import logging
 from pathlib import Path
 from typing import List, Optional, Union, Tuple, Sequence
 
-from credsweeper import DiffContentProvider
-from credsweeper.config import Config
+from credsweeper.config.config import Config
 from credsweeper.file_handler.abstract_provider import AbstractProvider
+from credsweeper.file_handler.content_provider import ContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
 from credsweeper.file_handler.text_content_provider import TextContentProvider
 
@@ -30,7 +30,7 @@ class FilesProvider(AbstractProvider):
         super().__init__(paths)
         self.skip_ignored = skip_ignored
 
-    def get_scannable_files(self, config: Config) -> Sequence[Union[DiffContentProvider, TextContentProvider]]:
+    def get_scannable_files(self, config: Config) -> Sequence[ContentProvider]:
         """Get list of full text file object for analysis of files with parent paths from "paths".
 
         Args:
@@ -40,7 +40,7 @@ class FilesProvider(AbstractProvider):
             preprocessed file objects for analysis
 
         """
-        text_content_provider_list: List[Union[DiffContentProvider, TextContentProvider]] = []
+        text_content_provider_list: List[ContentProvider] = []
         for path in self.paths:
             if isinstance(path, (str, Path)):
                 new_files = FilePathExtractor.get_file_paths(config, path)

credsweeper/file_handler/patches_provider.py

@@ -4,12 +4,12 @@ from pathlib import Path
 from typing import List, Union, Tuple, Sequence
 
 from credsweeper.common.constants import DiffRowType
-from credsweeper.config import Config
+from credsweeper.config.config import Config
 from credsweeper.file_handler.abstract_provider import AbstractProvider
+from credsweeper.file_handler.content_provider import ContentProvider
 from credsweeper.file_handler.diff_content_provider import DiffContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
-from credsweeper.file_handler.text_content_provider import TextContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 
@@ -50,17 +50,16 @@ class PatchesProvider(AbstractProvider):
 
         return raw_patches
 
-    def get_files_sequence(self,
-                           raw_patches: List[List[str]]) -> Sequence[Union[DiffContentProvider, TextContentProvider]]:
+    def get_files_sequence(self, raw_patches: List[List[str]]) -> Sequence[ContentProvider]:
         """Returns sequence of files"""
-        files: List[Union[DiffContentProvider, TextContentProvider]] = []
+        files: List[ContentProvider] = []
         for raw_patch in raw_patches:
-            files_data = Util.patch2files_diff(raw_patch, self.change_type)
+            files_data = DiffContentProvider.patch2files_diff(raw_patch, self.change_type)
             for file_path, file_diff in files_data.items():
                 files.append(DiffContentProvider(file_path=file_path, change_type=self.change_type, diff=file_diff))
         return files
 
-    def get_scannable_files(self, config: Config) -> Sequence[Union[DiffContentProvider, TextContentProvider]]:
+    def get_scannable_files(self, config: Config) -> Sequence[ContentProvider]:
         """Get files to scan. Output based on the `paths` field.
 
         Args:

credsweeper/file_handler/text_content_provider.py

@@ -6,7 +6,7 @@ from typing import List, Optional, Union, Tuple, Generator
 
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.file_handler.content_provider import ContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/filters/__init__.py

@@ -1,5 +1,3 @@
-from credsweeper.filters.filter import Filter  # isort:skip
-
 from credsweeper.filters.line_git_binary_check import LineGitBinaryCheck
 from credsweeper.filters.line_specific_key_check import LineSpecificKeyCheck
 from credsweeper.filters.line_uue_part_check import LineUUEPartCheck
@@ -12,11 +10,11 @@ from credsweeper.filters.value_base64_data_check import ValueBase64DataCheck
 from credsweeper.filters.value_base64_encoded_pem_check import ValueBase64EncodedPem
 from credsweeper.filters.value_base64_key_check import ValueBase64KeyCheck
 from credsweeper.filters.value_base64_part_check import ValueBase64PartCheck
+from credsweeper.filters.value_basic_auth_check import ValueBasicAuthCheck
 from credsweeper.filters.value_blocklist_check import ValueBlocklistCheck
 from credsweeper.filters.value_camel_case_check import ValueCamelCaseCheck
 from credsweeper.filters.value_couple_keyword_check import ValueCoupleKeywordCheck
 from credsweeper.filters.value_dictionary_keyword_check import ValueDictionaryKeywordCheck
-from credsweeper.filters.value_dictionary_value_length_check import ValueDictionaryValueLengthCheck
 from credsweeper.filters.value_discord_bot_check import ValueDiscordBotCheck
 from credsweeper.filters.value_entropy_base32_check import ValueEntropyBase32Check
 from credsweeper.filters.value_entropy_base36_check import ValueEntropyBase36Check
@@ -30,6 +28,7 @@ from credsweeper.filters.value_jfrog_token_check import ValueJfrogTokenCheck
 from credsweeper.filters.value_json_web_key_check import ValueJsonWebKeyCheck
 from credsweeper.filters.value_json_web_token_check import ValueJsonWebTokenCheck
 from credsweeper.filters.value_last_word_check import ValueLastWordCheck
+from credsweeper.filters.value_length_check import ValueLengthCheck
 from credsweeper.filters.value_method_check import ValueMethodCheck
 from credsweeper.filters.value_not_allowed_pattern_check import ValueNotAllowedPatternCheck
 from credsweeper.filters.value_not_part_encoded_check import ValueNotPartEncodedCheck

credsweeper/filters/filter.py

@@ -1,7 +1,8 @@
 from abc import abstractmethod, ABC
+from typing import Optional
 
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 
 
@@ -9,7 +10,8 @@ class Filter(ABC):
     """Base class for all filters that operates on 'line_data' objects."""
 
     @abstractmethod
-    def __init__(self, config: Config, *args):
+    def __init__(self, config: Optional[Config], *args):
+        """Config is optional for a filter"""
         raise NotImplementedError()
 
     @abstractmethod

credsweeper/filters/group/__init__.py

@@ -1,5 +1,3 @@
-from credsweeper.filters.group.group import Group  # isort:skip
-
 from credsweeper.filters.group.general_keyword import GeneralKeyword
 from credsweeper.filters.group.general_pattern import GeneralPattern
 from credsweeper.filters.group.password_keyword import PasswordKeyword

credsweeper/filters/group/general_keyword.py

@@ -1,7 +1,7 @@
 from credsweeper.common.constants import GroupType
-from credsweeper.config import Config
+from credsweeper.config.config import Config
 from credsweeper.filters import ValueDictionaryKeywordCheck
-from credsweeper.filters.group import Group
+from credsweeper.filters.group.group import Group
 
 
 class GeneralKeyword(Group):

credsweeper/filters/group/general_pattern.py

@@ -1,6 +1,6 @@
 from credsweeper.common.constants import GroupType
-from credsweeper.config import Config
-from credsweeper.filters.group import Group
+from credsweeper.config.config import Config
+from credsweeper.filters.group.group import Group
 
 
 class GeneralPattern(Group):