credsweeper 1.12.1__tar.gz → 1.12.2__tar.gz

{credsweeper-1.12.1 → credsweeper-1.12.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.12.1
+Version: 1.12.2
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues

{credsweeper-1.12.1 → credsweeper-1.12.2}/credsweeper/__init__.py

@@ -24,4 +24,4 @@ __all__ = [
     "__version__"
 ]
 
-__version__ = "1.12.1"
+__version__ = "1.12.2"

{credsweeper-1.12.1 → credsweeper-1.12.2}/credsweeper/__main__.py

@@ -372,9 +372,11 @@ def drill(args: Namespace) -> Tuple[int, int]:
         # then - credsweeper
         credsweeper = get_credsweeper(args)
         # use flat iterations to avoid recursive limits
-        to_scan = list(commits_sha1)
+        to_scan = set(commits_sha1)
         # local speedup for already scanned commits - avoid file system interactive
         scanned = set()
+        # to avoid double-check
+        skipped = set()
         while to_scan:
             commit_sha1 = to_scan.pop()
             if commit_sha1 in scanned:
@@ -382,8 +384,8 @@ def drill(args: Namespace) -> Tuple[int, int]:
                 continue
             commit = repo.commit(commit_sha1)
             if commit.parents:
-                # add parents anyway
-                to_scan.extend(x.hexsha for x in commit.parents)
+                # add parents only when they were not skipped or scanned previously
+                to_scan.update(x.hexsha for x in commit.parents if x.hexsha not in skipped and x.hexsha not in scanned)
             # check whether the commit has been checked and the report is present
             skip_already_scanned = False
             if args.json_filename:
@@ -401,9 +403,10 @@ def drill(args: Namespace) -> Tuple[int, int]:
                 else:
                     credsweeper.xlsx_filename = xlsx_path
             if skip_already_scanned:
-                logger.info("Skip already scanned commit: %s", commit_sha1)
+                skipped.add(commit_sha1)
+                logger.info("Skip already scanned commit: %s %s", commit_sha1, commit.committed_datetime.isoformat())
                 continue
-            logger.info("Scan commit: %s", commit_sha1)
+            logger.info("Scan commit: %s %s", commit_sha1, commit.committed_datetime.isoformat())
             # prepare all files to scan in the commit with bytes->IO transformation to avoid a multiprocess issue
             if providers := get_commit_providers(commit, repo):
                 credsweeper.credential_manager.candidates.clear()

{credsweeper-1.12.1 → credsweeper-1.12.2}/credsweeper/common/morpheme_checklist.txt

@@ -227,6 +227,7 @@ bind
 bio
 bipol
 bit
+bixby
 black
 blan
 bless
@@ -1440,6 +1441,7 @@ topic
 tory
 total
 touch
+tour
 trace
 tract
 traffic

{credsweeper-1.12.1 → credsweeper-1.12.2}/credsweeper/deep_scanner/deep_scanner.py

@@ -150,7 +150,7 @@ class DeepScanner(
                 deep_scanners.append(XmlScanner)
                 fallback_scanners.append(ByteScanner)
         elif Util.is_eml(data):
-            if ".eml" == descriptor.extension:
+            if descriptor.extension in (".eml", ".mht"):
                 deep_scanners.append(EmlScanner)
             else:
                 if 0 < depth:

{credsweeper-1.12.1 → credsweeper-1.12.2}/credsweeper/filters/value_camel_case_check.py

@@ -12,7 +12,7 @@ from credsweeper.utils.util import Util
 class ValueCamelCaseCheck(Filter):
     """Check that candidate is not written in camel case."""
 
-    CAMEL_CASE = ["^([a-z]+([A-Z][a-z]+)+)$", "^([A-Z][a-z]+([A-Z][a-z]+)+)$"]
+    CAMEL_CASE = ["[a-z]+([A-Z][a-z]+)+", "[A-Z][a-z]+([A-Z][a-z]+)+"]
     CAMEL_CASE_PATTERN = re.compile(Util.get_regex_combine_or(CAMEL_CASE))
 
     def __init__(self, config: Optional[Config] = None) -> None:
@@ -31,7 +31,7 @@ class ValueCamelCaseCheck(Filter):
         """
         if line_data.is_well_quoted_value:
             return False
-        if self.CAMEL_CASE_PATTERN.match(line_data.value):
+        if self.CAMEL_CASE_PATTERN.fullmatch(line_data.value):
             return static_keyword_checklist.check_morphemes(line_data.value.lower(), 1)
 
         return False

{credsweeper-1.12.1 → credsweeper-1.12.2}/credsweeper/rules/config.yaml

@@ -278,7 +278,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>dt0[A-Za-z]{1}[0-9]{2}\.[0-9A-Z]{24}\.[0-9A-Z]{64})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - dt0
   min_line_len: 90
@@ -308,7 +308,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>[0-9]{12,18}\|[0-9A-Za-z_-]{24,28})(?![0-9A-Za-z_+-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - "|"
   required_regex: "[0-9A-Za-z_/+-]{15}"
@@ -323,7 +323,7 @@
   type: pattern
   values:
     - (?i)((git)[0-9A-Za-z_-]{0,80}(token|key|api)[0-9A-Za-z_-]{0,80}(\s)*(=|:|:=)(\s)*(["']?)(?P<value>[0-9a-z]{40})(["']?))
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   use_ml: true
   required_substrings:
     - git
@@ -338,7 +338,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>AIza[0-9A-Za-z_-]{35})
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - AIza
   min_line_len: 39
@@ -367,7 +367,7 @@
   type: pattern
   values:
     - (?P<value>GOCSPX-[0-9A-Za-z_-]{28})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - GOCSPX-
   min_line_len: 40
@@ -381,7 +381,7 @@
   type: pattern
   values:
     - (?P<value>ya29\.[0-9A-Za-z_-]{22,8000})
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - ya29.
   min_line_len: 27
@@ -395,7 +395,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>1//0[0-9A-Za-z_-]{80,8000})
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - 1//0
   min_line_len: 84
@@ -409,7 +409,7 @@
   type: pattern
   values:
     - (?P<value>HRKU-([0-9A-Za-z_-]{60}|[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}))
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - HRKU-
   min_line_len: 41
@@ -423,7 +423,7 @@
   type: pattern
   values:
     - (?P<value>IGQVJ[=0-9A-Za-z_-]{100,8000})(?![=0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - IGQVJ
   min_line_len: 105
@@ -494,7 +494,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>[0-9A-Za-z_-]{32}-us[0-9]{1,2})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - -us
   min_line_len: 35
@@ -593,7 +593,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>SG\.[0-9A-Za-z_-]{16,32}\.[0-9A-Za-z_-]{16,64})
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - SG.
   min_line_len: 34
@@ -620,10 +620,11 @@
   confidence: strong
   type: pattern
   values:
-    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>xox[a-z]\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+    - (?P<value>(xapp|xox[a-z])\-[0-9A-Za-z-]{10,250})(?![0-9A-Za-z_-])
+  filter_type: TokenPattern
   required_substrings:
     - xox
+    - xapp
   min_line_len: 15
   target:
     - code
@@ -681,7 +682,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>sq0[a-z]{3}-[0-9A-Za-z_-]{22}([0-9A-Za-z_-]{21})?)(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - sq0
   min_line_len: 29
@@ -803,7 +804,7 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>[0-9]{8,10}:[0-9A-Za-z_-]{35})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - :AA
   min_line_len: 45
@@ -817,7 +818,7 @@
   type: pattern
   values:
     - (?P<value>pypi-[0-9A-Za-z_-]{150,255})
-  filter_type: GeneralPattern
+  filter_type: TokenPattern
   required_substrings:
     - pypi-
   min_line_len: 155
@@ -825,6 +826,20 @@
     - code
     - doc
 
+- name: NPM Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>npm_[0-9A-Za-z_-]{36,255})
+  filter_type: TokenPattern
+  required_substrings:
+    - npm_
+  min_line_len: 40
+  target:
+    - code
+    - doc
+
 - name: Github Classic Token
   severity: high
   confidence: strong
@@ -1541,6 +1556,22 @@
     - code
     - doc
 
+- name: Postman Credentials
+  severity: medium
+  confidence: moderate
+  type: pattern
+  values:
+    - (?P<value>(PMAK-[0-9a-f]{24}-[0-9a-f]{34}|PMAT-[0-9A-Z]{26}))
+  min_line_len: 29
+  filter_type:
+    - ValuePatternCheck
+  required_substrings:
+    - PMAK-
+    - PMAT-
+  target:
+    - code
+    - doc
+
 - name: Basic Authorization
   severity: medium
   confidence: strong

{credsweeper-1.12.1 → credsweeper-1.12.2}/credsweeper/utils/util.py

@@ -61,11 +61,11 @@ class Util:
     def get_shannon_entropy(data: Union[str, bytes]) -> float:
         """Borrowed from http://blog.dkbza.org/2007/05/scanning-data-for-entropy-anomalies.html."""
         if not data:
-            return 0.
+            return 0.0
         size = len(data)
         _uniq, counts = np.unique(list(data), return_counts=True)
         probabilities = counts / size
-        entropy = float(-np.sum(probabilities * np.log2(probabilities)))
+        entropy = -float(np.sum(probabilities * np.log2(probabilities)))
         return entropy
 
     # Precalculated data for speedup