credsweeper 1.11.4__tar.gz → 1.11.6__tar.gz

{credsweeper-1.11.4 → credsweeper-1.11.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.11.4
+Version: 1.11.6
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues
@@ -37,6 +37,7 @@ Requires-Dist: python-dateutil
 Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Requires-Dist: pyyaml
+Requires-Dist: rpmfile
 Requires-Dist: whatthepatch
 Requires-Dist: xlrd
 Description-Content-Type: text/markdown
@@ -140,11 +141,7 @@ cat output.json
                 "value_start": 12,
                 "value_end": 19,
                 "variable": "password",
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 2.120589933192232,
-                    "valid": false
-                }
+                "entropy": 2.12059
             }
         ]
     }

{credsweeper-1.11.4 → credsweeper-1.11.6}/README.md

@@ -97,11 +97,7 @@ cat output.json
                 "value_start": 12,
                 "value_end": 19,
                 "variable": "password",
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 2.120589933192232,
-                    "valid": false
-                }
+                "entropy": 2.12059
             }
         ]
     }

credsweeper-1.11.6/credsweeper/__init__.py

@@ -0,0 +1,27 @@
+from credsweeper.app import CredSweeper
+from credsweeper.common.constants import ThresholdPreset, Severity, Confidence
+from credsweeper.file_handler.byte_content_provider import ByteContentProvider
+from credsweeper.file_handler.content_provider import ContentProvider
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.diff_content_provider import DiffContentProvider
+from credsweeper.file_handler.string_content_provider import StringContentProvider
+from credsweeper.file_handler.text_content_provider import TextContentProvider
+
+from credsweeper.ml_model.ml_validator import MlValidator
+
+__all__ = [
+    "ByteContentProvider",  #
+    "Confidence",  #
+    "ContentProvider",  #
+    "CredSweeper",  #
+    "DataContentProvider",  #
+    "DiffContentProvider",  #
+    "MlValidator",  #
+    "Severity",  #
+    "StringContentProvider",  #
+    "TextContentProvider",  #
+    "ThresholdPreset",  #
+    "__version__"
+]
+
+__version__ = "1.11.6"

{credsweeper-1.11.4 → credsweeper-1.11.6}/credsweeper/__main__.py

@@ -5,16 +5,19 @@ import sys
 import time
 from argparse import ArgumentParser, ArgumentTypeError, Namespace, BooleanOptionalAction
 from pathlib import Path
-from typing import Any, Union, Dict
+from typing import Any, Union, Dict, Tuple, Sequence
+
+from git import Repo, Commit
 
 from credsweeper import __version__
 from credsweeper.app import APP_PATH, CredSweeper
 from credsweeper.common.constants import ThresholdPreset, Severity, RuleType, DiffRowType, ML_HUNK
 from credsweeper.file_handler.abstract_provider import AbstractProvider
+from credsweeper.file_handler.byte_content_provider import ByteContentProvider
 from credsweeper.file_handler.files_provider import FilesProvider
 from credsweeper.file_handler.patches_provider import PatchesProvider
 from credsweeper.logger.logger import Logger
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 EXIT_SUCCESS = 0
 EXIT_FAILURE = 1
@@ -118,6 +121,11 @@ def get_arguments() -> Namespace:
                        const="log.yaml",
                        dest="export_log_config",
                        metavar="PATH")
+    group.add_argument("--git", help="git repo to scan", dest="git", metavar="PATH")
+    parser.add_argument("--ref",
+                        help="scan git repo from the ref, otherwise - all branches were scanned (slow)",
+                        dest="ref",
+                        type=str)
     parser.add_argument("--rules",
                         help="path of rule config file (default: credsweeper/rules/config.yaml). "
                         f"severity:{[i.value for i in Severity]} "
@@ -246,8 +254,8 @@ def get_arguments() -> Namespace:
                         default=False)
     parser.add_argument("--log",
                         "-l",
-                        help=f"provide logging level of {list(Logger.LEVELS.keys())}"
-                        f"(default: 'warning', case insensitive)",
+                        help=(f"provide logging level of {list(Logger.LEVELS.keys())}"
+                              f" (default: 'warning', case insensitive)"),
                         default="warning",
                         dest="log",
                         metavar="LOG_LEVEL",
@@ -268,6 +276,39 @@ def get_arguments() -> Namespace:
     return parser.parse_args()
 
 
+def get_credsweeper(args: Namespace) -> CredSweeper:
+    """Common function to create the instance"""
+    if args.denylist_path is not None:
+        denylist = [line for line in Util.read_file(args.denylist_path) if line]
+    else:
+        denylist = []
+    return CredSweeper(rule_path=args.rule_path,
+                       config_path=args.config_path,
+                       json_filename=args.json_filename,
+                       xlsx_filename=args.xlsx_filename,
+                       stdout=args.stdout,
+                       color=args.color,
+                       hashed=args.hashed,
+                       subtext=args.subtext,
+                       sort_output=args.sort_output,
+                       use_filters=args.no_filters,
+                       pool_count=args.jobs,
+                       ml_batch_size=args.ml_batch_size,
+                       ml_threshold=args.ml_threshold,
+                       ml_config=args.ml_config,
+                       ml_model=args.ml_model,
+                       ml_providers=args.ml_providers,
+                       find_by_ext=args.find_by_ext,
+                       depth=args.depth,
+                       doc=args.doc,
+                       severity=args.severity,
+                       size_limit=args.size_limit,
+                       exclude_lines=denylist,
+                       exclude_values=denylist,
+                       thrifty=args.thrifty,
+                       log_level=args.log)
+
+
 def scan(args: Namespace, content_provider: AbstractProvider) -> int:
     """Scan content_provider data, print results or save them to json_filename is not None
 
@@ -283,42 +324,101 @@ def scan(args: Namespace, content_provider: AbstractProvider) -> int:
 
     """
     try:
-        if args.denylist_path is not None:
-            denylist = [line for line in Util.read_file(args.denylist_path) if line]
-        else:
-            denylist = []
-
-        credsweeper = CredSweeper(rule_path=args.rule_path,
-                                  config_path=args.config_path,
-                                  json_filename=args.json_filename,
-                                  xlsx_filename=args.xlsx_filename,
-                                  stdout=args.stdout,
-                                  color=args.color,
-                                  hashed=args.hashed,
-                                  subtext=args.subtext,
-                                  sort_output=args.sort_output,
-                                  use_filters=args.no_filters,
-                                  pool_count=args.jobs,
-                                  ml_batch_size=args.ml_batch_size,
-                                  ml_threshold=args.ml_threshold,
-                                  ml_config=args.ml_config,
-                                  ml_model=args.ml_model,
-                                  ml_providers=args.ml_providers,
-                                  find_by_ext=args.find_by_ext,
-                                  depth=args.depth,
-                                  doc=args.doc,
-                                  severity=args.severity,
-                                  size_limit=args.size_limit,
-                                  exclude_lines=denylist,
-                                  exclude_values=denylist,
-                                  thrifty=args.thrifty,
-                                  log_level=args.log)
+        credsweeper = get_credsweeper(args)
         return credsweeper.run(content_provider=content_provider)
     except Exception as exc:
         logger.critical(exc, exc_info=True)
+        logger.exception(exc)
     return -1
 
 
+def get_commit_providers(commit: Commit, repo: Repo) -> Sequence[ByteContentProvider]:
+    """Process a commit and for providers"""
+    result = {}
+    ancestors = commit.parents or [repo.tree()]
+    for parent in ancestors:
+        for diff in parent.diff(commit):
+            # only result files
+            blob_b = diff.b_blob
+            if blob_b and blob_b.path not in result:
+                try:
+                    result[blob_b.path] = ByteContentProvider(content=blob_b.data_stream.read(),
+                                                              file_path=str(blob_b.path),
+                                                              info=DiffRowType.ADDED.value)
+                except Exception as exc:
+                    logger.warning(f"A submodule was not properly initialized or commit was removed: {exc}")
+    return list(result.values())
+
+
+def drill(args: Namespace) -> Tuple[int, int]:
+    """Scan repository for branches and commits
+    Returns:
+        total credentials found
+        total scanned commits
+    """
+    total_credentials = 0
+    total_commits = 0
+    try:
+        # repo init first
+        repo = Repo(args.git)
+        if args.ref:
+            commits_sha1 = set(x.commit.hexsha for x in repo.refs if x.name == args.ref)
+            if not commits_sha1:
+                commits_sha1 = {args.ref}  # single commit sha1 reference
+        else:
+            commits_sha1 = set(x.commit.hexsha for x in repo.refs
+                               if x.name.startswith('origin/') or x.name.startswith('refs/heads/'))
+        logger.info(f"Git repository {args.git} with commits: {commits_sha1}")
+        # then - credsweeper
+        credsweeper = get_credsweeper(args)
+        # use flat iterations to avoid recursive limits
+        to_scan = list(commits_sha1)
+        # local speedup for already scanned commits - avoid file system interactive
+        scanned = set()
+        while to_scan:
+            commit_sha1 = to_scan.pop()
+            if commit_sha1 in scanned:
+                # the commit was scanned in this launch
+                continue
+            commit = repo.commit(commit_sha1)
+            if commit.parents:
+                # add parents anyway
+                to_scan.extend(x.hexsha for x in commit.parents)
+            # check whether the commit has been checked and the report is present
+            skip_already_scanned = False
+            if args.json_filename:
+                json_path = Path(args.json_filename)
+                json_path = json_path.with_suffix(f".{commit_sha1}{json_path.suffix}")
+                if json_path.exists():
+                    skip_already_scanned = True
+                else:
+                    credsweeper.json_filename = json_path
+            if args.xlsx_filename:
+                xlsx_path = Path(args.xlsx_filename)
+                xlsx_path = xlsx_path.with_suffix(f".{commit_sha1}{xlsx_path.suffix}")
+                if xlsx_path.exists():
+                    skip_already_scanned = True
+                else:
+                    credsweeper.xlsx_filename = xlsx_path
+            if skip_already_scanned:
+                logger.info("Skip already scanned commit: %s", commit_sha1)
+                continue
+            logger.info("Scan commit: %s", commit_sha1)
+            # prepare all files to scan in the commit with bytes->IO transformation to avoid a multiprocess issue
+            if providers := get_commit_providers(commit, repo):
+                credsweeper.credential_manager.candidates.clear()
+                credsweeper.scan(providers)
+                credsweeper.post_processing()
+                credsweeper.export_results()
+                total_credentials += credsweeper.credential_manager.len_credentials()
+            total_commits += 1
+            scanned.add(commit_sha1)
+    except Exception as exc:
+        logger.critical(exc, exc_info=True)
+        return -1, total_commits
+    return total_credentials, total_commits
+
+
 def main() -> int:
     """Main function"""
     result = EXIT_FAILURE
@@ -328,7 +428,7 @@ def main() -> int:
     if args.banner:
         print(f"CredSweeper {__version__} crc32:{check_integrity():08x}")
     Logger.init_logging(args.log, args.log_config_path)
-    logger.info(f"Init CredSweeper object with arguments: {args}")
+    logger.info(f"Init CredSweeper object with arguments: {args} CWD: {os.getcwd()}")
     summary: Dict[str, int] = {}
     if args.path:
         logger.info(f"Run analyzer on path: {args.path}")
@@ -353,6 +453,12 @@ def main() -> int:
             result = EXIT_SUCCESS
             # collect number of all found credential to produce error code when necessary
             credentials_number = add_credentials_number + del_credentials_number
+    elif args.git:
+        logger.info(f"Run analyzer on GIT: {args.git}")
+        credentials_number, commits_number = drill(args)
+        summary[f"Detected Credentials in {args.git} for {commits_number} commits "] = credentials_number
+        if 0 <= credentials_number:
+            result = EXIT_SUCCESS
     elif args.export_config:
         logging.info(f"Exporting default config to file: {args.export_config}")
         config_dict = Util.json_load(APP_PATH / "secret" / "config.json")

{credsweeper-1.11.4 → credsweeper-1.11.6}/credsweeper/app.py

@@ -11,18 +11,18 @@ from colorama import Style
 # Directory of credsweeper sources MUST be placed before imports to avoid circular import error
 APP_PATH = Path(__file__).resolve().parent
 
+from credsweeper.scanner.scanner import Scanner
 from credsweeper.common.constants import Severity, ThresholdPreset, DiffRowType, DEFAULT_ENCODING
-from credsweeper.config import Config
-from credsweeper.credentials import Candidate, CredentialManager, CandidateKey
+from credsweeper.config.config import Config
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.credentials.candidate_key import CandidateKey
+from credsweeper.credentials.credential_manager import CredentialManager
 from credsweeper.deep_scanner.deep_scanner import DeepScanner
 from credsweeper.file_handler.content_provider import ContentProvider
-from credsweeper.file_handler.diff_content_provider import DiffContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
 from credsweeper.file_handler.abstract_provider import AbstractProvider
-from credsweeper.file_handler.text_content_provider import TextContentProvider
-from credsweeper.scanner import Scanner
 from credsweeper.ml_model.ml_validator import MlValidator
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 
@@ -215,7 +215,7 @@ class CredSweeper:
             content_provider: path objects to scan
 
         """
-        _empty_list: Sequence[Union[DiffContentProvider, TextContentProvider]] = []
+        _empty_list: Sequence[ContentProvider] = []
         file_extractors = content_provider.get_scannable_files(self.config) if content_provider else _empty_list
         if not file_extractors:
             logger.info(f"No scannable targets for {len(content_provider.paths)} paths")
@@ -229,7 +229,7 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    def scan(self, content_providers: Sequence[Union[DiffContentProvider, TextContentProvider]]) -> None:
+    def scan(self, content_providers: Sequence[ContentProvider]) -> None:
         """Run scanning of files from an argument "content_providers".
 
         Args:
@@ -243,7 +243,7 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    def __single_job_scan(self, content_providers: Sequence[Union[DiffContentProvider, TextContentProvider]]) -> None:
+    def __single_job_scan(self, content_providers: Sequence[ContentProvider]) -> None:
         """Performs scan in main thread"""
         logger.info(f"Scan for {len(content_providers)} providers")
         all_cred = self.files_scan(content_providers)
@@ -251,7 +251,7 @@ class CredSweeper:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    def __multi_jobs_scan(self, content_providers: Sequence[Union[DiffContentProvider, TextContentProvider]]) -> None:
+    def __multi_jobs_scan(self, content_providers: Sequence[ContentProvider]) -> None:
         """Performs scan with multiple jobs"""
         # use this separation to satisfy YAPF formatter
         yapfix = "%(asctime)s | %(levelname)s | %(processName)s:%(threadName)s | %(filename)s:%(lineno)s | %(message)s"
@@ -265,7 +265,7 @@ class CredSweeper:
         logger.info(f"Scan in {pool_count} processes for {len(content_providers)} providers")
         with multiprocessing.get_context("spawn").Pool(processes=pool_count,
                                                        initializer=CredSweeper.pool_initializer,
-                                                       initargs=(log_kwargs, )) as pool:
+                                                       initargs=(log_kwargs,)) as pool:  # yapf: disable
             try:
                 for scan_results in pool.imap_unordered(self.files_scan,
                                                         (content_providers[x::pool_count] for x in range(pool_count))):

{credsweeper-1.11.4 → credsweeper-1.11.6}/credsweeper/config/config.py

@@ -4,7 +4,7 @@ from typing import Dict, List, Optional, Set, Any
 from humanfriendly import parse_size
 
 from credsweeper.common.constants import Severity, DEFAULT_PATTERN_LEN
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 
 class Config:

{credsweeper-1.11.4 → credsweeper-1.11.6}/credsweeper/credentials/augment_candidates.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 
 
 def augment_candidates(candidates: List[Candidate], new_candidates: List[Candidate]):

{credsweeper-1.11.4 → credsweeper-1.11.6}/credsweeper/credentials/candidate.py

@@ -4,7 +4,7 @@ from json.encoder import py_encode_basestring_ascii
 from typing import Any, Dict, List, Optional
 
 from credsweeper.common.constants import Severity, Confidence
-from credsweeper.config import Config
+from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
 
 

{credsweeper-1.11.4 → credsweeper-1.11.6}/credsweeper/credentials/credential_manager.py

@@ -2,7 +2,7 @@ import logging
 from multiprocessing import Manager
 from typing import List, Dict, Tuple
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.credentials.candidate_group_generator import CandidateGroupGenerator, CandidateKey
 
 logger = logging.getLogger(__name__)

{credsweeper-1.11.4 → credsweeper-1.11.6}/credsweeper/credentials/line_data.py

@@ -8,8 +8,8 @@ from typing import Any, Dict, Optional, Tuple
 from colorama import Fore, Style
 
 from credsweeper.common.constants import MAX_LINE_LENGTH, UTF_8, StartEnd, ML_HUNK
-from credsweeper.config import Config
-from credsweeper.utils import Util
+from credsweeper.config.config import Config
+from credsweeper.utils.util import Util
 
 
 class LineData: