credsweeper 1.12.0__tar.gz → 1.12.2__tar.gz

{credsweeper-1.12.0 → credsweeper-1.12.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.12.0
+Version: 1.12.2
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues

{credsweeper-1.12.0 → credsweeper-1.12.2}/credsweeper/__init__.py

@@ -24,4 +24,4 @@ __all__ = [
     "__version__"
 ]
 
-__version__ = "1.12.0"
+__version__ = "1.12.2"

{credsweeper-1.12.0 → credsweeper-1.12.2}/credsweeper/__main__.py

@@ -372,9 +372,11 @@ def drill(args: Namespace) -> Tuple[int, int]:
         # then - credsweeper
         credsweeper = get_credsweeper(args)
         # use flat iterations to avoid recursive limits
-        to_scan = list(commits_sha1)
+        to_scan = set(commits_sha1)
         # local speedup for already scanned commits - avoid file system interactive
         scanned = set()
+        # to avoid double-check
+        skipped = set()
         while to_scan:
             commit_sha1 = to_scan.pop()
             if commit_sha1 in scanned:
@@ -382,8 +384,8 @@ def drill(args: Namespace) -> Tuple[int, int]:
                 continue
             commit = repo.commit(commit_sha1)
             if commit.parents:
-                # add parents anyway
-                to_scan.extend(x.hexsha for x in commit.parents)
+                # add parents only when they were not skipped or scanned previously
+                to_scan.update(x.hexsha for x in commit.parents if x.hexsha not in skipped and x.hexsha not in scanned)
             # check whether the commit has been checked and the report is present
             skip_already_scanned = False
             if args.json_filename:
@@ -401,9 +403,10 @@ def drill(args: Namespace) -> Tuple[int, int]:
                 else:
                     credsweeper.xlsx_filename = xlsx_path
             if skip_already_scanned:
-                logger.info("Skip already scanned commit: %s", commit_sha1)
+                skipped.add(commit_sha1)
+                logger.info("Skip already scanned commit: %s %s", commit_sha1, commit.committed_datetime.isoformat())
                 continue
-            logger.info("Scan commit: %s", commit_sha1)
+            logger.info("Scan commit: %s %s", commit_sha1, commit.committed_datetime.isoformat())
             # prepare all files to scan in the commit with bytes->IO transformation to avoid a multiprocess issue
             if providers := get_commit_providers(commit, repo):
                 credsweeper.credential_manager.candidates.clear()

{credsweeper-1.12.0 → credsweeper-1.12.2}/credsweeper/common/morpheme_checklist.txt

@@ -227,6 +227,7 @@ bind
 bio
 bipol
 bit
+bixby
 black
 blan
 bless
@@ -1440,6 +1441,7 @@ topic
 tory
 total
 touch
+tour
 trace
 tract
 traffic

{credsweeper-1.12.0 → credsweeper-1.12.2}/credsweeper/deep_scanner/deep_scanner.py

@@ -150,7 +150,7 @@ class DeepScanner(
                 deep_scanners.append(XmlScanner)
                 fallback_scanners.append(ByteScanner)
         elif Util.is_eml(data):
-            if ".eml" == descriptor.extension:
+            if descriptor.extension in (".eml", ".mht"):
                 deep_scanners.append(EmlScanner)
             else:
                 if 0 < depth:

{credsweeper-1.12.0 → credsweeper-1.12.2}/credsweeper/filters/value_camel_case_check.py

@@ -12,7 +12,7 @@ from credsweeper.utils.util import Util
 class ValueCamelCaseCheck(Filter):
     """Check that candidate is not written in camel case."""
 
-    CAMEL_CASE = ["^([a-z]+([A-Z][a-z]+)+)$", "^([A-Z][a-z]+([A-Z][a-z]+)+)$"]
+    CAMEL_CASE = ["[a-z]+([A-Z][a-z]+)+", "[A-Z][a-z]+([A-Z][a-z]+)+"]
     CAMEL_CASE_PATTERN = re.compile(Util.get_regex_combine_or(CAMEL_CASE))
 
     def __init__(self, config: Optional[Config] = None) -> None:
@@ -31,7 +31,7 @@ class ValueCamelCaseCheck(Filter):
         """
         if line_data.is_well_quoted_value:
             return False
-        if self.CAMEL_CASE_PATTERN.match(line_data.value):
+        if self.CAMEL_CASE_PATTERN.fullmatch(line_data.value):
             return static_keyword_checklist.check_morphemes(line_data.value.lower(), 1)
 
         return False

{credsweeper-1.12.0 → credsweeper-1.12.2}/credsweeper/rules/config.yaml

@@ -278,7 +278,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>dt0[A-Za-z]{1}[0-9]{2}\.[0-9A-Z]{24}\.[0-9A-Z]{64})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - dt0
   min_line_len: 90
@@ -308,7 +308,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>[0-9]{12,18}\|[0-9A-Za-z_-]{24,28})(?![0-9A-Za-z_+-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - "|"
   required_regex: "[0-9A-Za-z_/+-]{15}"
@@ -323,7 +323,7 @@
   type: pattern
   values:
     - (?i)((git)[0-9A-Za-z_-]{0,80}(token|key|api)[0-9A-Za-z_-]{0,80}(\s)*(=|:|:=)(\s)*(["']?)(?P<value>[0-9a-z]{40})(["']?))
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   use_ml: true
   required_substrings:
     - git
@@ -338,7 +338,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>AIza[0-9A-Za-z_-]{35})
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - AIza
   min_line_len: 39
@@ -367,7 +367,7 @@
   type: pattern
   values:
     - (?P<value>GOCSPX-[0-9A-Za-z_-]{28})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - GOCSPX-
   min_line_len: 40
@@ -381,7 +381,7 @@
   type: pattern
   values:
     - (?P<value>ya29\.[0-9A-Za-z_-]{22,8000})
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - ya29.
   min_line_len: 27
@@ -389,13 +389,27 @@
     - code
     - doc
 
+- name: Google OAuth Refresh Token
+  severity: medium
+  confidence: weak
+  type: pattern
+  values:
+    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>1//0[0-9A-Za-z_-]{80,8000})
+  filter_type: TokenPattern
+  required_substrings:
+    - 1//0
+  min_line_len: 84
+  target:
+    - code
+    - doc
+
 - name: Heroku Credentials
   severity: high
   confidence: strong
   type: pattern
   values:
     - (?P<value>HRKU-([0-9A-Za-z_-]{60}|[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}))
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - HRKU-
   min_line_len: 41
@@ -409,7 +423,7 @@
   type: pattern
   values:
     - (?P<value>IGQVJ[=0-9A-Za-z_-]{100,8000})(?![=0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - IGQVJ
   min_line_len: 105
@@ -480,7 +494,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>[0-9A-Za-z_-]{32}-us[0-9]{1,2})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - -us
   min_line_len: 35
@@ -579,7 +593,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>SG\.[0-9A-Za-z_-]{16,32}\.[0-9A-Za-z_-]{16,64})
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - SG.
   min_line_len: 34
@@ -606,10 +620,11 @@
   confidence: strong
   type: pattern
   values:
-    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>xox[a-z]\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+    - (?P<value>(xapp|xox[a-z])\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
+  filter_type: TokenPattern
   required_substrings:
     - xox
+    - xapp
   min_line_len: 15
   target:
     - code
@@ -667,7 +682,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>sq0[a-z]{3}-[0-9A-Za-z_-]{22}([0-9A-Za-z_-]{21})?)(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - sq0
   min_line_len: 29
@@ -789,7 +804,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>[0-9]{8,10}:[0-9A-Za-z_-]{35})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - :AA
   min_line_len: 45
@@ -803,7 +818,7 @@
   type: pattern
   values:
     - (?P<value>pypi-[0-9A-Za-z_-]{150,255})
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - pypi-
   min_line_len: 155
@@ -811,6 +826,20 @@
     - code
     - doc
 
+- name: NPM Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>npm_[0-9A-Za-z_-]{36,255})
+  filter_type: TokenPattern
+  required_substrings:
+    - npm_
+  min_line_len: 40
+  target:
+    - code
+    - doc
+
 - name: Github Classic Token
   severity: high
   confidence: strong
@@ -1246,7 +1275,7 @@
     - (?P<value>[0-9A-Za-z_-]{14}\.atlasv1\.[0-9A-Za-z_-]{67})(?![0-9A-Za-z_-])
   filter_type:
     - ValuePatternCheck
-    - ValueEntropyBase64Check
+    - ValueCoupleKeywordCheck(3)
   min_line_len: 90
   required_substrings:
     - .atlasv1.
@@ -1280,18 +1309,19 @@
     - code
     - doc
 
-- name: Jira 2FA
+- name: OTP / 2FA Secret
   severity: info
   confidence: weak
   type: pattern
   values:
-    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>[A-Z2-7]{16})(?![=0-9A-Za-z_+-])
+    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>([A-Z2-7]{16}){1,2})(?![=0-9A-Za-z_+-])
   filter_type:
     - ValueCoupleKeywordCheck
     - ValuePatternCheck
     - ValueEntropyBase32Check
     - ValueBase32DataCheck
     - ValueTokenBase32Check
+    - ValueBase64PartCheck
   min_line_len: 16
   required_regex: "[0-9A-Za-z_/+-]{15}"
   target:
@@ -1303,11 +1333,11 @@
   confidence: strong
   type: pattern
   values:
-    - (?P<value>sk-[0-9A-Za-z_-]{16,32}(T3BlbkFJ|9wZW5BS|PcGVuQU)[0-9A-Za-z_-]{16,32})
+    - (?P<value>sk-[0-9A-Za-z_-]{16,160}(T3BlbkFJ|9wZW5BS|PcGVuQU)[0-9A-Za-z_-]{16,160})
   min_line_len: 51
   filter_type:
     - ValuePatternCheck
-    - ValueEntropyBase64Check
+    - ValueCoupleKeywordCheck
   required_substrings:
     - T3BlbkFJ
     - 9wZW5BS
@@ -1325,7 +1355,7 @@
   min_line_len: 36
   filter_type:
     - ValuePatternCheck
-    - ValueEntropyBase64Check
+    - ValueCoupleKeywordCheck
   required_substrings:
     - dckr_pat_
     - dckr_oat_
@@ -1341,6 +1371,7 @@
     - (?P<value>SWMTKN-1-[0-9a-z]{50}-[0-9a-z]{25})
   min_line_len: 85
   filter_type:
+    - ValuePatternCheck
     - ValueCoupleKeywordCheck
   required_substrings:
     - SWMTKN-1-
@@ -1348,6 +1379,22 @@
     - code
     - doc
 
+- name: Docker Swarm Key
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?P<value>SWMKEY-1-[0-9A-Za-z]{43})
+  min_line_len: 52
+  filter_type:
+    - ValuePatternCheck
+    - ValueCoupleKeywordCheck(3)
+  required_substrings:
+    - SWMKEY-1-
+  target:
+    - code
+    - doc
+
 - name: Groq API Key
   severity: high
   confidence: strong
@@ -1355,11 +1402,29 @@
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>gsk_[0-9A-Za-z_-]{52})(?![0-9A-Za-z_-])
   min_line_len: 56
+  filter_type:
+    - ValuePatternCheck
+    - ValueCoupleKeywordCheck
+  required_substrings:
+    - WGdyb3FY
+    - hncm9xW
+    - YZ3JvcV
+  target:
+    - code
+    - doc
+
+- name: X AI API Key
+  severity: high
+  confidence: moderate
+  type: pattern
+  values:
+    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>xai-[0-9A-Za-z_-]{80})(?![0-9A-Za-z_-])
+  min_line_len: 84
   filter_type:
     - ValuePatternCheck
     - ValueEntropyBase64Check
   required_substrings:
-    - gsk_
+    - xai-
   target:
     - code
     - doc
@@ -1389,7 +1454,6 @@
   min_line_len: 37
   filter_type:
     - ValuePatternCheck
-    - ValueEntropyBase64Check
   required_substrings:
     - tvly-
   target:
@@ -1492,6 +1556,22 @@
     - code
     - doc
 
+- name: Postman Credentials
+  severity: medium
+  confidence: moderate
+  type: pattern
+  values:
+    - (?P<value>(PMAK-[0-9a-f]{24}-[0-9a-f]{34}|PMAT-[0-9A-Z]{26}))
+  min_line_len: 29
+  filter_type:
+    - ValuePatternCheck
+  required_substrings:
+    - PMAK-
+    - PMAT-
+  target:
+    - code
+    - doc
+
 - name: Basic Authorization
   severity: medium
   confidence: strong

{credsweeper-1.12.0 → credsweeper-1.12.2}/credsweeper/utils/util.py

@@ -61,11 +61,11 @@ class Util:
     def get_shannon_entropy(data: Union[str, bytes]) -> float:
         """Borrowed from http://blog.dkbza.org/2007/05/scanning-data-for-entropy-anomalies.html."""
         if not data:
-            return 0.
+            return 0.0
         size = len(data)
         _uniq, counts = np.unique(list(data), return_counts=True)
         probabilities = counts / size
-        entropy = float(-np.sum(probabilities * np.log2(probabilities)))
+        entropy = -float(np.sum(probabilities * np.log2(probabilities)))
         return entropy
 
     # Precalculated data for speedup