credsweeper 1.10.5__tar.gz → 1.10.7__tar.gz

{credsweeper-1.10.5 → credsweeper-1.10.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.10.5
+Version: 1.10.7
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues
@@ -87,7 +87,7 @@ Full documentation can be found here: <https://credsweeper.readthedocs.io/>
 
 ### Main Requirements
 
-- Python 3.8, 3.9, 3.10, 3.11, 3.12
+- Python 3.9, 3.10, 3.11, 3.12
 
 ### Installation
 

{credsweeper-1.10.5 → credsweeper-1.10.7}/README.md

@@ -46,7 +46,7 @@ Full documentation can be found here: <https://credsweeper.readthedocs.io/>
 
 ### Main Requirements
 
-- Python 3.8, 3.9, 3.10, 3.11, 3.12
+- Python 3.9, 3.10, 3.11, 3.12
 
 ### Installation
 

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.10.5"
+__version__ = "1.10.7"

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/__main__.py

@@ -3,7 +3,7 @@ import logging
 import os
 import sys
 import time
-from argparse import ArgumentParser, ArgumentTypeError, Namespace
+from argparse import ArgumentParser, ArgumentTypeError, Namespace, BooleanOptionalAction
 from typing import Any, Union, Dict
 
 from credsweeper import __version__
@@ -205,12 +205,16 @@ def get_arguments() -> Namespace:
                         metavar="POSITIVE_INT")
     parser.add_argument("--thrifty",
                         help="clear objects after scan to reduce memory consumption",
-                        action="store_const",
-                        const=True)
+                        action=BooleanOptionalAction,
+                        default=True)
     parser.add_argument("--skip_ignored",
                         help="parse .gitignore files and skip credentials from ignored objects",
                         dest="skip_ignored",
                         action="store_true")
+    parser.add_argument("--error",
+                        help="produce error code if credentials are found",
+                        action=BooleanOptionalAction,
+                        default=False)
     parser.add_argument("--save-json",
                         nargs="?",
                         help="save result to json file (default: output.json)",
@@ -223,16 +227,21 @@ def get_arguments() -> Namespace:
                         const="output.xlsx",
                         dest="xlsx_filename",
                         metavar="PATH")
-    parser.add_argument("--color", "-C", help="print results with colorization", action="store_const", const=True)
+    parser.add_argument("--stdout", help="print results to stdout", action=BooleanOptionalAction, default=True)
+    parser.add_argument("--color", help="print results with colorization", action=BooleanOptionalAction, default=False)
     parser.add_argument("--hashed",
                         help="line, variable, value will be hashed in output",
-                        action="store_const",
-                        const=True)
+                        action=BooleanOptionalAction,
+                        default=False)
     parser.add_argument("--subtext",
                         help=f"line text will be stripped in {2 * ML_HUNK} symbols but value and variable are kept",
-                        action="store_const",
-                        const=True)
-    parser.add_argument("--sort", help="enable output sorting", dest="sort_output", action="store_true")
+                        action=BooleanOptionalAction,
+                        default=False)
+    parser.add_argument("--sort",
+                        help="enable output sorting",
+                        dest="sort_output",
+                        action=BooleanOptionalAction,
+                        default=False)
     parser.add_argument("--log",
                         "-l",
                         help=f"provide logging level of {list(Logger.LEVELS.keys())}"
@@ -281,6 +290,7 @@ def scan(args: Namespace, content_provider: AbstractProvider) -> int:
                                   config_path=args.config_path,
                                   json_filename=args.json_filename,
                                   xlsx_filename=args.xlsx_filename,
+                                  stdout=args.stdout,
                                   color=args.color,
                                   hashed=args.hashed,
                                   subtext=args.subtext,
@@ -310,6 +320,7 @@ def scan(args: Namespace, content_provider: AbstractProvider) -> int:
 def main() -> int:
     """Main function"""
     result = EXIT_FAILURE
+    credentials_number = 0
     start_time = time.time()
     args = get_arguments()
     if args.banner:
@@ -336,15 +347,20 @@ def main() -> int:
         del_credentials_number = scan(args, content_provider)
         summary["Deleted File Credentials"] = del_credentials_number
         if 0 <= add_credentials_number and 0 <= del_credentials_number:
+            # it means the scan was successful done
             result = EXIT_SUCCESS
+            # collect number of all found credential to produce error code when necessary
+            credentials_number = add_credentials_number + del_credentials_number
     elif args.export_config:
         logging.info(f"Exporting default config to file: {args.export_config}")
         config_dict = Util.json_load(APP_PATH / "secret" / "config.json")
         Util.json_dump(config_dict, args.export_config)
+        result = EXIT_SUCCESS
     elif args.export_log_config:
         logging.info(f"Exporting default logger config to file: {args.export_log_config}")
         config_dict = Util.yaml_load(APP_PATH / "secret" / "log.yaml")
         Util.yaml_dump(config_dict, args.export_log_config)
+        result = EXIT_SUCCESS
     elif args.banner and 2 == len(sys.argv):
         # only extend version invocation
         result = EXIT_SUCCESS
@@ -357,6 +373,10 @@ def main() -> int:
         end_time = time.time()
         print(f"Time Elapsed: {end_time - start_time}s")
 
+    if args.error and EXIT_SUCCESS == result and 0 < credentials_number:
+        # override result when credentials were found with the requirement
+        result = EXIT_FAILURE
+
     return result
 
 

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/app.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import multiprocessing
 import signal
@@ -41,6 +42,7 @@ class CredSweeper:
                  config_path: Optional[str] = None,
                  json_filename: Union[None, str, Path] = None,
                  xlsx_filename: Union[None, str, Path] = None,
+                 stdout: bool = False,
                  color: bool = False,
                  hashed: bool = False,
                  subtext: bool = False,
@@ -70,7 +72,8 @@ class CredSweeper:
                 default built-in config is used if None
             json_filename: optional string variable, path to save result to json
             xlsx_filename: optional string variable, path to save result to xlsx
-            color: print results to stdout with colorization
+            stdout: print results to stdout
+            color: print concise results to stdout with colorization
             hashed: use hash of line, value and variable instead plain text
             subtext: use subtext of line near variable-value like it performed in ML
             use_filters: boolean variable, specifying the need of rule filters
@@ -110,6 +113,7 @@ class CredSweeper:
         self.credential_manager = CredentialManager()
         self.json_filename: Union[None, str, Path] = json_filename
         self.xlsx_filename: Union[None, str, Path] = xlsx_filename
+        self.stdout = stdout
         self.color = color
         self.hashed = hashed
         self.subtext = subtext
@@ -245,8 +249,7 @@ class CredSweeper:
         # PatchesProvider has the attribute. Circular import error appears with using the isinstance
         change_type = content_provider.change_type if hasattr(content_provider, "change_type") else None
         self.export_results(change_type)
-
-        return len(self.credential_manager.get_credentials())
+        return self.credential_manager.len_credentials()
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
@@ -392,7 +395,6 @@ class CredSweeper:
         Args:
             change_type: flag to know which file should be created for a patch
         """
-        is_exported = False
 
         credentials = self.credential_manager.get_credentials()
 
@@ -410,15 +412,22 @@ class CredSweeper:
 
         if self.json_filename:
             json_path = Path(self.json_filename)
-            is_exported = True
             if isinstance(change_type, DiffRowType):
                 # add suffix for appropriated reports to create two files for the patch scan
                 json_path = json_path.with_suffix(f".{change_type.value}{json_path.suffix}")
-            Util.json_dump([credential.to_json(hashed=self.hashed, subtext=self.subtext) for credential in credentials],
-                           file_path=json_path)
+            with open(json_path, 'w') as f:
+                # use the approach to reduce total memory usage in case of huge data
+                first_item = True
+                f.write('[\n')
+                for credential in credentials:
+                    if first_item:
+                        first_item = False
+                    else:
+                        f.write(",\n")
+                    f.write(json.dumps(credential.to_json(hashed=self.hashed, subtext=self.subtext), indent=4))
+                f.write("\n]")
 
         if self.xlsx_filename:
-            is_exported = True
             data_list = []
             for credential in credentials:
                 data_list.extend(credential.to_dict_list(hashed=self.hashed, subtext=self.subtext))
@@ -434,7 +443,6 @@ class CredSweeper:
                 df.to_excel(self.xlsx_filename, sheet_name="report", index=False)
 
         if self.color:
-            is_exported = True
             for credential in credentials:
                 for line_data in credential.line_data_list:
                     # bright rule name and path or info
@@ -443,6 +451,6 @@ class CredSweeper:
                           Style.RESET_ALL)
                     print(line_data.get_colored_line(hashed=self.hashed, subtext=self.subtext))
 
-        if is_exported is False:
+        if self.stdout:
             for credential in credentials:
                 print(credential.to_str(hashed=self.hashed, subtext=self.subtext))

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/common/keyword_pattern.py

@@ -3,27 +3,30 @@ import re
 
 class KeywordPattern:
     """Pattern set of keyword types"""
-    key_left = r"(\\[nrt]|%[0-9a-f]{2})?"\
-               r"(?P<variable>(([`'\"]+[^:='\"`}<>\\/&?]*|[^:='\"`}<>\s()\\/&?;,%]*)" \
+    key_left = r"(\\[nrt]|%[0-9a-f]{2})?" \
+               r"(?P<variable>(([`'\"]{1,8}[^:='\"`}<>\\/&?]*|[^:='\"`}<>\s()\\/&?;,%]*)" \
                r"(?P<keyword>"
     # there will be inserted a keyword
     key_right = r")" \
-                r"(&(quot|apos);|[^%:='\"`<>{?!&]*)[`'\"]*))"  # <variable>
-    separator = r"(\s|\\+[tnr])*\]?(\s|\\+[tnr])*" \
-                r"(?P<separator>:( [a-z]{3,9}[?]? )?=|:|=(>|&gt;|\\u0026gt;)|!=|===|==|=|%3d)" \
-                r"(\s|\\+[tnr])*"
+                r"[^%:='\"`<>{?!&]*" \
+                r")" \
+                r"(&(quot|apos);|%[0-9a-f]{2}|[`'\"])*" \
+                r")"  # <variable>
+    separator = r"(\s|\\{1,8}[tnr])*\]?(\s|\\{1,8}[tnr])*" \
+                r"(?P<separator>:(\s[a-z]{3,9}[?]?\s)?=|:|=(>|&gt;|\\u0026gt;)|!==|!=|===|==|=|%3d)" \
+                r"(\s|\\{1,8}[tnr])*"
     # might be curly, square or parenthesis with words before
     wrap = r"(?P<wrap>(" \
-           r"(new(\s|\\+[tnr])+)?" \
+           r"(new(\s|\\{1,8}[tnr]){1,8})?" \
            r"([0-9a-z_.]|-(>|(&|\\\\*u0026)gt;))*" \
-           r"[\[\(\{]"\
-           r"(\s|\\+[tnr])*" \
+           r"[\[\(\{]" \
+           r"(\s|\\{1,8}[tnr])*" \
            r"([0-9a-z_]{1,32}=)?" \
-           r")+)?"
+           r"){1,8})?"
     string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[`'\"])))?"
     left_quote = r"(?P<value_leftquote>((?P<esq>\\{1,8})?([`'\"]|&(quot|apos);)){1,4}))?"
     # Authentication scheme ( oauth | basic | bearer | apikey ) precedes to credential
-    auth_keywords = r"( ?(oauth|bot|basic|bearer|apikey|accesskey) )?"
+    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey)\s)?"
     value = r"(?P<value>" \
             r"(?(value_leftquote)" \
             r"(" \
@@ -31,11 +34,15 @@ class KeywordPattern:
             r"(?(esq)((?!(?P=esq)([`'\"]|&(quot|apos);)).)|((?!(?P=value_leftquote)).)))" \
             r"|" \
             r"(?!&(quot|apos);)" \
-            r"(\\+([ tnr]|[^\s`'\"])|[^\s`'\",;\\])" \
-            r"){3,8000}" \
+            r"(\\{1,8}([ tnr]|[^\s`'\"])" \
+            r"|" \
+            r"(?P<url_esc>%[0-9a-f]{2})" \
+            r"|" \
+            r"(?(url_esc)[^\s`'\",;\\&]|[^\s`'\",;\\])" \
+            r")){3,8000}" \
             r"|(\{[^}]{3,8000}\})" \
             r"|(<[^>]{3,8000}>)" \
-            r")"
+            r")"  # <value>
     right_quote = r"(?(value_leftquote)" \
                   r"(?P<value_rightquote>(?<!\\)(?P=value_leftquote)|\\$|(?<=[0-9a-z+_/-])$)" \
                   r"|" \
@@ -44,7 +51,7 @@ class KeywordPattern:
     @classmethod
     def get_keyword_pattern(cls, keyword: str) -> re.Pattern:
         """Returns compiled regex pattern"""
-        expression = "".join([  #
+        expression = ''.join([  #
             cls.key_left,  #
             keyword,  #
             cls.key_right,  #

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/credentials/credential_manager.py

@@ -9,16 +9,24 @@ logger = logging.getLogger(__name__)
 
 
 class CredentialManager:
-    """The manager allows you to store, add and delete separate credit candidates.
-
-    Parameters:
-        candidates: list of credential candidates
-
-    """
+    """The manager allows you to store, add and delete separate credit candidates."""
 
     def __init__(self) -> None:
         self.candidates: List[Candidate] = list(Manager().list())
 
+    def clear_credentials(self) -> None:
+        """Clear credential candidates stored in the manager."""
+        self.candidates.clear()
+
+    def len_credentials(self) -> int:
+        """Get number of credential candidates stored in the manager.
+
+        Return:
+            Non-negative integer
+
+        """
+        return len(self.candidates)
+
     def get_credentials(self) -> List[Candidate]:
         """Get all credential candidates stored in the manager.
 

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/filters/value_allowlist_check.py

@@ -8,22 +8,35 @@ from credsweeper.utils import Util
 
 
 class ValueAllowlistCheck(Filter):
-    """Check that patterns from the list is not present in the candidate value."""
+    """Check that the patterns do not MATCH the candidate value."""
 
     ALLOWED = [
         r"ENC\(.*\)",  #
         r"ENC\[.*\]",  #
         r"\$\{(\*|[0-9]+|[a-z_].*)\}",  #
-        r"\$([0-9]+\b|[a-z_]+[0-9a-z_]*)",  #
+        r"\$[0-9]+(\s|$)",  #
         r"\$\$[a-z_]+(\^%[0-9a-z_]+)?",  #
-        r"#\{.*\}",  #
+        r"#\{.+\}",  # Ruby: String Interpolation
         r"\{\{.+\}\}",  #
-        r"\S{0,5}\*{5,}",  #
         r".*@@@hl@@@(암호|비번|PW|PASS)@@@endhl@@@",  #
     ]
 
     ALLOWED_PATTERN = re.compile(Util.get_regex_combine_or(ALLOWED), flags=re.IGNORECASE)
-    ALLOWED_UNQUOTED_PATTERN = re.compile(r"[~a-z0-9_]+((\.|->)[a-z0-9_]+)+\(.*$", flags=re.IGNORECASE)
+
+    ALLOWED_QUOTED = [
+        r"\$[a-z_]+[0-9a-z_]*([$\s]|$)",  #
+        r".*\*\*\*",  #
+    ]
+
+    ALLOWED_QUOTED_PATTERN = re.compile(Util.get_regex_combine_or(ALLOWED_QUOTED), flags=re.IGNORECASE)
+
+    ALLOWED_UNQUOTED = [
+        r"[~a-z0-9_]+((\.|->)[a-z0-9_]+)+\(.*$",  #
+        r"\$[a-z_]+[0-9a-z_]*\b",  #
+        r".*\*\*\*\*\*",  #
+    ]
+
+    ALLOWED_UNQUOTED_PATTERN = re.compile(Util.get_regex_combine_or(ALLOWED_UNQUOTED), flags=re.IGNORECASE)
 
     def __init__(self, config: Config = None) -> None:
         pass
@@ -42,8 +55,11 @@ class ValueAllowlistCheck(Filter):
 
         if self.ALLOWED_PATTERN.match(line_data.value):
             return True
-
-        if not line_data.is_well_quoted_value and self.ALLOWED_UNQUOTED_PATTERN.match(line_data.value):
-            return True
+        elif line_data.is_well_quoted_value:
+            if self.ALLOWED_QUOTED_PATTERN.match(line_data.value):
+                return True
+        else:
+            if self.ALLOWED_UNQUOTED_PATTERN.match(line_data.value):
+                return True
 
         return False

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/filters/value_blocklist_check.py

@@ -11,8 +11,11 @@ class ValueBlocklistCheck(Filter):
         "true",
         "false",
         "null",
+        "none",
         "bearer",
         "string",
+        "value",
+        "undefined",
     ]
 
     def __init__(self, config: Config = None) -> None:

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/filters/value_token_base_check.py

@@ -37,10 +37,9 @@ class ValueTokenBaseCheck(Filter):
 
     @staticmethod
     def get_ppf(n: int) -> float:
-        """
-        from scipy.stats import t
-        print('\n'.join(f'{n}: {t.ppf(0.9827, n-1):.8f},' for n in [8,10,15,16,20,24,25,32,40,50,64]))
-        """
+        """Code used to produce the values"""
+        # from scipy.stats import t
+        # print('\n'.join(f'{n}: {t.ppf(0.9827, n-1):.8f},' for n in [8,10,15,16,20,24,25,32,40,50,64]))
         return ValueTokenBaseCheck.MUL_DICT[n]
 
     def run(self, line_data: LineData, target: AnalysisTarget) -> bool:

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/rules/config.yaml

@@ -1391,6 +1391,22 @@
     - code
     - doc
 
+- name: Tavily API Key
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>tvly-[0-9A-Za-z_-]{32,40})(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+    - ValueEntropyBase64Check
+  required_substrings:
+    - tvly-
+  target:
+    - code
+    - doc
+
 - name: Discord Bot Token
   severity: high
   confidence: strong

{credsweeper-1.10.5 → credsweeper-1.10.7}/credsweeper/utils/hop_stat.py

@@ -25,11 +25,37 @@ class HopStat:
         ')': '0',
         '_': '-',
         '+': '=',
+        'Q': 'q',
+        'W': 'w',
+        'E': 'e',
+        'R': 'r',
+        'T': 't',
+        'Y': 'y',
+        'U': 'u',
+        'I': 'i',
+        'O': 'o',
+        'P': 'p',
         '{': '[',
         '}': ']',
         '|': '\\',
+        'A': 'a',
+        'S': 's',
+        'D': 'd',
+        'F': 'f',
+        'G': 'g',
+        'H': 'h',
+        'J': 'j',
+        'K': 'k',
+        'L': 'l',
         ':': ';',
         '"': "'",
+        'Z': 'z',
+        'X': 'x',
+        'C': 'c',
+        'V': 'v',
+        'B': 'b',
+        'N': 'n',
+        'M': 'm',
         '<': ',',
         '>': '.',
         '?': '/',
@@ -75,7 +101,7 @@ class HopStat:
 
         """
         hops = []
-        value = value.lower().translate(HopStat.TRANSLATION)
+        value = value.translate(HopStat.TRANSLATION)
         for a, b in zip(value[:-1], value[1:]):
             hop = self.__hop_dict.get((a, b))
             if hop is None: