credsweeper 1.12.2__py3-none-any.whl → 1.13.0__py3-none-any.whl

credsweeper/rules/config.yaml

@@ -3,7 +3,7 @@
   confidence: weak
   type: pattern
   values:
-    - (?P<variable>(\w*(?i:비밀번호|비번|패스워드|키|암호화?|토큰|(?<!by)pass(?!ed|ing|ion|es|age)|\bpwd?\b|token|secret|key|cred)\w*)\s*(설정은|[=:!]{1,3}))?\s*([._0-9A-Za-z\[\]]*get(env)?\s*\(\s*(?(variable)[^,]+|[\"'\\]*(\\*([\"']|&(quot|apos|#3[49]);)){0,4}(\w*(?i:(?<!by)pass(?!ed|ing|ion|es|age|\s+[a-z]{3,80})|\bpwd?\b|token|secret|key|cred)\w*))(\\*([\"']|&(quot|apos|#3[49]);)){0,4})\s*,\s*(default\s*=\s*)?([brufl@]{1,2}(?=\\*[\"'&]))?(?P<lq>(\\*([\"']|&(quot|apos|#3[49]);)){1,4})(?P<value>(.(?!(?P=lq))){4,80}.?)
+    - (?P<variable>(\w*(?i:비밀번호|비번|패스워드|키|암호화?|토큰|(?<!by)pass(?!e[dns]|ing|ion|age)|\bpwd?\b|token|secret|key|cred)\w*)\s*(설정은|[=:!]{1,3}))?\s*([._0-9A-Za-z\[\]]*get(env)?\s*\(\s*(?(variable)[^,]+|[\"'\\]*(\\*([\"']|&(quot|apos|#3[49]);)){0,4}(\w*(?i:(?<!by)pass(?!e[dns]|ing|ion|age|\s+[a-z]{3,80})|\bpwd?\b|token|secret|key|cred)\w*))(\\*([\"']|&(quot|apos|#3[49]);)){0,4})\s*,\s*(default\s*=\s*)?([brufl@]{1,2}(?=\\*[\"'&]))?(?P<lq>(\\*([\"']|&(quot|apos|#3[49]);)){1,4})(?P<value>(.(?!(?P=lq))){4,80}.?)
   filter_type:
     - ValueAllowlistCheck
     - LineGitBinaryCheck
@@ -34,7 +34,7 @@
   confidence: weak
   type: pattern
   values:
-    - (?P<wrap>[\"'`(])?\s*(?P<variable>(\w*(?i:(?<!by)passw?o?r?d?s?(?!ed|ing|ion|es|age)|pwd?\b|\bp/w\b|token|secret|key|credential)\w*|비밀번호|비번|패스워드|키|암호화?|토큰))[\"'`]*(\s+(?i:is|are|was|were)(\s*[:-])?\s+|\s*(설정은|[=:!]{1,3})\s*)(?P<quote>[\"'`]{1,6})?(?P<value>(?(quote)(?(wrap)[^\"'`)]{4,80}|[^\"'`]{4,80})|(?(wrap)[^\"'`)]{4,80}|\S{4,80})))
+    - (?P<wrap>[\"'`(])?\s*(?P<variable>(\w*(?i:(?<!by)passw?o?r?d?s?(?!e[dns]|ing|ion|age)|pwd?\b|\bp/w\b|token|secret|key|credential)\w*|비밀번호|비번|패스워드|키|암호화?|토큰))[\"'`]*(\s+(?i:is|are|was|were)(\s*[:-])?\s+|\s*(설정은|[=:!]{1,3})\s*)(?P<quote>[\"'`]{1,6})?(?P<value>(?(quote)(?(wrap)[^\"'`)]{4,80}|[^\"'`]{4,80})|(?(wrap)[^\"'`)]{4,80}|\S{4,80})))
   filter_type:
     - ValueAllowlistCheck
     - LineGitBinaryCheck
@@ -73,7 +73,7 @@
     - ValueAllowlistCheck
     - ValuePatternCheck(4)
     - ValueEntropyBase64Check
-    - ValueCoupleKeywordCheck
+    - ValueMorphemesCheck
   min_line_len: 16
   required_substrings:
     - token
@@ -90,7 +90,7 @@
   confidence: moderate
   type: pattern
   values:
-    - (?P<variable>[\"'`]?(?i:(?<!id[ :/])pa[as]swo?r?ds?|pwd?|p/w|비밀번호|비번|패스워드|암호)[\"'`]?)((\s)*[=:](\s)*)(?P<quote>[\"'`(])?(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){8,31}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x))(?(quote)[)\"'`])
+    - (?P<variable>[\"'`]?(?i:(?<!id[ :/])pa[as]swo?r?ds?|pwd?|p/w|비밀번호|비번|패스워드|암호)[\"'`]?)((\s)*[=:](\s)*)(?P<quote>[\"'`(])?(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){8,64}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x))(?(quote)[)\"'`])
   filter_type:
     - ValueAllowlistCheck
     - ValuePatternCheck(4)
@@ -118,7 +118,7 @@
   confidence: moderate
   type: pattern
   values:
-    - (^|\s|(?P<variable>(?i:\bip[\s/]{1,80}id[\s/]{1,80}pw[\s/:]{0,80}))|(?P<url>://))(?P<ip>(?<![0-9.])[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}(?![0-9.]))((\s*[(])?|(?(variable)[\s,/]{1,80}|(?(url)[,]|[,/])))\s*\w[\w.-]{3,80}[\s,/]{1,80}(?P<value>(?(url)(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9_+=~!@#$%^&*;?-])){7,31}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x)|(?-i:(?P<e>[A-Z])|(?P<f>[a-z])|(?P<g>[0-9/_+=~!@#$%^&*;?-])){7,31}(?(e)(?(f)(?(g)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x)))(?:\s|[^/]|$)
+    - (^|\s|(?P<variable>(?i:\bip[\s/]{1,80}id[\s/]{1,80}pw[\s/:]{0,80}))|(?P<url>://))(?P<ip>(?<![0-9.])[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}(?![0-9.]))((\s*[(])?|(?(variable)[\s,/]{1,80}|(?(url)[,]|[,/])))\s*\w[\w.-]{3,80}[\s,/]{1,80}(?P<value>(?(url)(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9_+=~!@#$%^&*;?-])){7,64}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x)|(?-i:(?P<e>[A-Z])|(?P<f>[a-z])|(?P<g>[0-9/_+=~!@#$%^&*;?-])){7,64}(?(e)(?(f)(?(g)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x)))(?:\s|[^/]|$)
   filter_type:
     - ValueAllowlistCheck
     - ValuePatternCheck(4)
@@ -134,7 +134,7 @@
   confidence: moderate
   type: pattern
   values:
-    - (?P<ddash>--)?(?P<variable>\w*(?i:pa[as]swords?|passwd?|pwd|\bp/w|\bpw|비밀번호|비번|패스워드|암호))\s*?(?(ddash)[ =]|[:=/>-]{1,2})\s*(?P<quote>[\"'`]{1,8})?(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){4,31}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x))(?(quote)(?P=quote)|(\s|$))
+    - (?P<ddash>--)?(?P<variable>\w*(?i:pa[as]swords?|passwd?|pwd|\bp/w|\bpw|비밀번호|비번|패스워드|암호))\s*?(?(ddash)[ =]|[:=/>-]{1,2})\s*(?P<quote>[\"'`]{1,8})?(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){4,64}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x))(?(quote)(?P=quote)|(\s|$))
     - (?P<ddash>--)?(?P<variable>(?i:user\s*)?(?i:id|login|account|root|admin|user|name|wifi|role|host|default|계정|아이디))\s*?(?(ddash)[ =]|[ :=])\s*?(?P<value>\S+)
   filter_type:
     - ValueAllowlistCheck
@@ -157,7 +157,7 @@
   confidence: moderate
   type: pattern
   values:
-    - (?P<variable>[\w.-]{0,80}(?i:(?P<id>\bid\b)|id\b|user|name|계정|아이디)[\w.-]{0,80}(?(id)[ :(/]{1,80}|[:(/]{1,80})(?i:pa[as]swo?r?ds?|pwd?|비밀번호|비번|패스워드|암호))\)?(\s*->\s*|[ =:)(/]{1,80}|\s+is\s+|\s+are\s+|\s*는\s*|\s*은\s*|\s*설정은\s*)\(?(?P<id_value>[\w.-]{2,31})[ :\(/\"',]{1,80}(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){4,31}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x))
+    - (?P<variable>[\w.-]{0,80}(?i:(?P<id>\bid\b)|id\b|user|name|계정|아이디)[\w.-]{0,80}(?(id)[ :(/]{1,80}|[:(/]{1,80})(?i:pa[as]swo?r?ds?|pwd?|비밀번호|비번|패스워드|암호))\)?(\s*->\s*|[ =:)(/]{1,80}|\s+is\s+|\s+are\s+|\s*는\s*|\s*은\s*|\s*설정은\s*)\(?(?P<id_value>[\w.-]{2,64})[ :\(/\"',]{1,80}(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){4,64}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x))
   filter_type:
     - ValueAllowlistCheck
     - ValuePatternCheck(4)
@@ -174,24 +174,6 @@
   target:
     - doc
 
-- name: SQL Password
-  severity: medium
-  confidence: weak
-  type: pattern
-  values:
-    - (\\[nrt]|\b)(?i:(?P<variable>(CREATE|ALTER|SET\s{1,8}PASSWORD|INSERT(\s{1,8}IGNORE)?|UPDATE\s{1,8}[^\s;]{1,80})\s{1,8}(LOGIN|USER|ROLE|FOR|INTO|SET)\s{1,8}([^\s;]{1,80}\s{1,8}|VALUES\s*\(){1,8}(IDENTIFIED((\s{1,8}WITH\s{1,8}\S{1,80})?\s{1,8}(BY|AS))|(=|WITH)?\s*PASSWORD\b(\s*=)?)))\s*(?P<wrap>[(]\s*)?(?P<value_leftquote>((?P<esq>\\{1,8})?([\"'`]|&(quot|apos|#3[49]);)){1,4})?(?P<value>(?(value_leftquote)((?!(?P=value_leftquote))(?(esq)((?!(?P=esq)([\"'`]|&(quot|apos|#3[49]);)).)|((?!(?P=value_leftquote)).)))|(?!&(quot|apos|#3[49]);)(\\+([ tnr]|[^\s\"'`])|[^\s\"'`,;\\])){3,80})(?(value_leftquote)(?P<value_rightquote>(?<!\\)(?P=value_leftquote))|(?(wrap)[)]|[\s\"'`,;]))
-  filter_type:
-    - ValueAllowlistCheck
-    - ValuePatternCheck
-  min_line_len: 8
-  required_substrings:
-    - password
-    - identified
-  target:
-    - doc
-    - code
-  use_ml: true
-
 - name: UUID
   severity: info
   confidence: strong
@@ -249,7 +231,7 @@
     - LineSpecificKeyCheck
     - ValuePatternCheck
     - ValueBase64PartCheck
-    - ValueCoupleKeywordCheck(3)
+    - ValueMorphemesCheck
   required_substrings:
     - A
   min_line_len: 20
@@ -317,21 +299,6 @@
     - code
     - doc
 
-- name: Github Old Token
-  severity: high
-  confidence: moderate
-  type: pattern
-  values:
-    - (?i)((git)[0-9A-Za-z_-]{0,80}(token|key|api)[0-9A-Za-z_-]{0,80}(\s)*(=|:|:=)(\s)*(["']?)(?P<value>[0-9a-z]{40})(["']?))
-  filter_type: TokenPattern
-  use_ml: true
-  required_substrings:
-    - git
-  min_line_len: 47
-  target:
-    - code
-    - doc
-
 - name: Google API Key
   severity: high
   confidence: moderate
@@ -480,7 +447,7 @@
     - (?P<variable>\b[dk])[^0-9A-Za-z_-]{1,8}(?P<value>[0-9A-Za-z_-]{22,8000})(?![=0-9A-Za-z_-])
   filter_type:
     - ValuePatternCheck
-    - ValueCoupleKeywordCheck(3)
+    - ValueMorphemesCheck
   required_substrings:
     - kty
   min_line_len: 8
@@ -507,10 +474,9 @@
   confidence: moderate
   type: pattern
   values:
-    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>key-[0-9A-Za-z_-]{32})(?![0-9A-Za-z_-])
-  filter_type: GeneralPattern
-  required_substrings:
-    - key-
+    - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>key-[0-9a-f]{32}|[0-9a-f]{32}-[0-9a-f]{8}-[0-9a-f]{8})(?![0-9A-Za-z_-])
+  filter_type: TokenPattern
+  required_regex: "[0-9A-Za-z_/+-]{15}"
   min_line_len: 36
   target:
     - code
@@ -728,76 +694,6 @@
     - code
     - doc
 
-- name: CMD ConvertTo-SecureString
-  severity: high
-  confidence: moderate
-  type: pattern
-  values:
-    - (?P<variable>ConvertTo-SecureString(\s\s*-(String|AsPlainText|Force))*)\s\s*(?P<value_leftquote>(\\?[\"']){1,3})?(?P<value>(?(value_leftquote)[^\"'\\]|[^\s\"'\\]){4,800})(?(value_leftquote)(?P<value_rightquote>(\\?[\"']){1,3}))
-  filter_type: GeneralKeyword
-  use_ml: true
-  required_substrings:
-    - convertto-securestring
-  min_line_len: 27
-  target:
-    - code
-
-- name: CMD Password
-  severity: high
-  confidence: moderate
-  type: pattern
-  values:
-    - (^|\W|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<variable>-[A-Za-z_-]*(?i:pass(in|out|word|phrase)))(\s|\\?[\"'],)\s*(?!-)(?P<value_leftquote>(\\?[\"']){1,3})?(pass:)?(?!file:|env:|fd:)(?P<value>(?(value_leftquote)[^\"'\\]|[^\s\"'\\]){4,80})(?(value_leftquote)(?P<value_rightquote>(\\?[\"']){1,3}))
-  filter_type: GeneralKeyword
-  use_ml: true
-  required_substrings:
-    - pass
-  min_line_len: 12
-  target:
-    - code
-
-- name: CMD Token
-  severity: high
-  confidence: moderate
-  type: pattern
-  values:
-    - (^|\W|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<variable>-[A-Za-z_-]*(?i:token))(\s|\\?[\"'],)\s*(?!-)(?P<value_leftquote>(\\?[\"']){1,3})?(?P<value>(?(value_leftquote)[^\"'\\]|[^\s\"'\\]){4,4000})(?(value_leftquote)(?P<value_rightquote>(\\?[\"']){1,3}))
-  filter_type: GeneralKeyword
-  use_ml: true
-  required_substrings:
-    - token
-  min_line_len: 12
-  target:
-    - code
-
-- name: CMD Secret
-  severity: high
-  confidence: moderate
-  type: pattern
-  values:
-    - (^|\W|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<variable>-[A-Za-z_-]*(?i:secret)[A-Za-z_-]*)(\s|\\?[\"'],)\s*(?!-)(?P<value_leftquote>(\\?[\"']){1,3})?(pass:)?(?!file:|env:|fd:)(?P<value>(?(value_leftquote)[^\"'\\]|[^\s\"'\\]){4,4000})(?(value_leftquote)(?P<value_rightquote>(\\?[\"']){1,3}))
-  filter_type: GeneralKeyword
-  use_ml: true
-  required_substrings:
-    - secret
-  min_line_len: 12
-  target:
-    - code
-
-- name: URL Credentials
-  severity: high
-  confidence: moderate
-  type: pattern
-  values:
-    - (?P<value_leftquote>[\"'])?(?P<variable>[+0-9A-Za-z-]{2,80}://)([^\s\'"<>\[\]^~`{|}:/]{0,80}:){1,3}(?P<value>[^\s\'"<>\[\]^~`{|}@:/]{3,80})@[^\s\'"<>\[\]^~`{|}@:/]{1,800}\\{0,8}(?P<value_rightquote>[\"'])?
-  filter_type: UrlCredentialsGroup
-  use_ml: true
-  required_substrings:
-    - ://
-  min_line_len: 10
-  target:
-    - code
-
 - name: Telegram Bot API Token
   severity: high
   confidence: moderate
@@ -832,7 +728,8 @@
   type: pattern
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>npm_[0-9A-Za-z_-]{36,255})
-  filter_type: TokenPattern
+  filter_type:
+    - ValueGitHubCheck
   required_substrings:
     - npm_
   min_line_len: 40
@@ -1086,12 +983,13 @@
   confidence: strong
   type: pattern
   values:
-    - (?P<value>do[op]_v1_[a-f0-9]{64})(?![0-9A-Za-z_-])
+    - (?P<value>do[opr]_v1_[a-f0-9]{64})(?![0-9A-Za-z_-])
   filter_type: TokenPattern
   min_line_len: 71
   required_substrings:
     - doo_v1_
     - dop_v1_
+    - dor_v1_
   target:
     - code
     - doc
@@ -1275,7 +1173,7 @@
     - (?P<value>[0-9A-Za-z_-]{14}\.atlasv1\.[0-9A-Za-z_-]{67})(?![0-9A-Za-z_-])
   filter_type:
     - ValuePatternCheck
-    - ValueCoupleKeywordCheck(3)
+    - ValueMorphemesCheck
   min_line_len: 90
   required_substrings:
     - .atlasv1.
@@ -1291,7 +1189,7 @@
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>S[ACNOPUX][A-Z2-7]{40,200})(?![=0-9A-Za-z_+-])
   min_line_len: 42
   filter_type:
-    - ValueCoupleKeywordCheck
+    - ValueMorphemesCheck
     - ValuePatternCheck
     - ValueEntropyBase32Check
     - ValueBase32DataCheck
@@ -1316,7 +1214,7 @@
   values:
     - (?:^|[^0-9A-Za-z_+-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>([A-Z2-7]{16}){1,2})(?![=0-9A-Za-z_+-])
   filter_type:
-    - ValueCoupleKeywordCheck
+    - ValueMorphemesCheck
     - ValuePatternCheck
     - ValueEntropyBase32Check
     - ValueBase32DataCheck
@@ -1337,7 +1235,7 @@
   min_line_len: 51
   filter_type:
     - ValuePatternCheck
-    - ValueCoupleKeywordCheck
+    - ValueMorphemesCheck
   required_substrings:
     - T3BlbkFJ
     - 9wZW5BS
@@ -1355,7 +1253,7 @@
   min_line_len: 36
   filter_type:
     - ValuePatternCheck
-    - ValueCoupleKeywordCheck
+    - ValueMorphemesCheck
   required_substrings:
     - dckr_pat_
     - dckr_oat_
@@ -1372,7 +1270,7 @@
   min_line_len: 85
   filter_type:
     - ValuePatternCheck
-    - ValueCoupleKeywordCheck
+    - ValueMorphemesCheck
   required_substrings:
     - SWMTKN-1-
   target:
@@ -1388,7 +1286,7 @@
   min_line_len: 52
   filter_type:
     - ValuePatternCheck
-    - ValueCoupleKeywordCheck(3)
+    - ValueMorphemesCheck
   required_substrings:
     - SWMKEY-1-
   target:
@@ -1404,7 +1302,7 @@
   min_line_len: 56
   filter_type:
     - ValuePatternCheck
-    - ValueCoupleKeywordCheck
+    - ValueMorphemesCheck
   required_substrings:
     - WGdyb3FY
     - hncm9xW
@@ -1515,7 +1413,7 @@
   values:
     - (?P<variable>discord(?:app)?\.com/api/webhooks)(?P<value>/[0-9]{16,22}/[0-9A-Za-z_-]{40,100})
   filter_type:
-    - ValueCoupleKeywordCheck(3)
+    - ValueMorphemesCheck
   required_substrings:
     - discordapp.com/api/webhooks
     - discord.com/api/webhooks
@@ -1602,6 +1500,109 @@
     - code
     - doc
 
+- name: SQL Password
+  severity: medium
+  confidence: weak
+  type: pattern
+  values:
+    - (\\[nrt]|\b)(?i:(?P<variable>(CREATE|ALTER|SET\s{1,8}PASSWORD|INSERT(\s{1,8}IGNORE)?|UPDATE\s{1,8}[^\s;]{1,80})\s{1,8}(LOGIN|USER|ROLE|FOR|INTO|SET)\s{1,8}([^\s;]{1,80}\s{1,8}|VALUES\s*\(){1,8}(IDENTIFIED((\s{1,8}WITH\s{1,8}\S{1,80})?\s{1,8}(BY|AS))|(=|WITH)?\s*PASSWORD\b(\s*=)?)))\s*(?P<wrap>[(]\s*)?(?P<value_leftquote>((?P<esq>\\{1,8})?([\"'`]|&(quot|apos|#3[49]);)){1,4})?(?P<value>(?(value_leftquote)((?!(?P=value_leftquote))(?(esq)((?!(?P=esq)([\"'`]|&(quot|apos|#3[49]);)).)|((?!(?P=value_leftquote)).)))|(?!&(quot|apos|#3[49]);)(\\+([ tnr]|[^\s\"'`])|[^\s\"'`,;\\])){3,80})(?(value_leftquote)(?P<value_rightquote>(?<!\\)(?P=value_leftquote))|(?(wrap)[)]|[\s\"'`,;]))
+  filter_type:
+    - ValueAllowlistCheck
+    - ValuePatternCheck
+  use_ml: true
+  min_line_len: 8
+  required_substrings:
+    - password
+    - identified
+  target:
+    - doc
+    - code
+
+- name: CURL User Password
+  severity: high
+  confidence: moderate
+  type: pattern
+  values:
+    - (?P<variable>curl)\s.*(-[uU]|--(proxy-)?user)\s\s*(?P<value_leftquote>(\\*[\"']){1,3})?(?(value_leftquote)[^\"'\\:]|[^\s\"'\\:]){0,64}:(?P<value>(?(value_leftquote)[^\"'\\]|[^\s\"'\\]){4,64})(?(value_leftquote)(?P<value_rightquote>(\\?[\"']){1,3}))
+  filter_type: GeneralKeyword
+  use_ml: true
+  required_substrings:
+    - curl
+  min_line_len: 16
+  target:
+    - code
+
+- name: CMD ConvertTo-SecureString
+  severity: high
+  confidence: moderate
+  type: pattern
+  values:
+    - (?P<variable>ConvertTo-SecureString(\s\s*-(String|AsPlainText|Force))*)\s\s*(?P<value_leftquote>(\\?[\"']){1,3})?(?P<value>(?(value_leftquote)[^\"'\\]|[^\s\"'\\]){4,800})(?(value_leftquote)(?P<value_rightquote>(\\?[\"']){1,3}))
+  filter_type: GeneralKeyword
+  use_ml: true
+  required_substrings:
+    - convertto-securestring
+  min_line_len: 27
+  target:
+    - code
+
+- name: CMD Password
+  severity: high
+  confidence: moderate
+  type: pattern
+  values:
+    - (^|\W|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<variable>-[A-Za-z_-]*(?i:pass(in|out|word|phrase)))(\s|\\?[\"'],)\s*(?!-)(?P<value_leftquote>(\\?[\"']){1,3})?(pass:)?(?!file:|env:|fd:)(?P<value>(?(value_leftquote)[^\"'\\]|[^\s\"'\\]){4,80})(?(value_leftquote)(?P<value_rightquote>(\\?[\"']){1,3}))
+  filter_type: GeneralKeyword
+  use_ml: true
+  required_substrings:
+    - pass
+  min_line_len: 12
+  target:
+    - code
+
+- name: CMD Token
+  severity: high
+  confidence: moderate
+  type: pattern
+  values:
+    - (^|\W|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<variable>-[A-Za-z_-]*(?i:token|oauth2-bearer))(\s|\\?[\"'],)\s*(?!-)(?P<value_leftquote>(\\?[\"']){1,3})?(?P<value>(?(value_leftquote)[^\"'\\]|[^\s\"'\\]){4,4000})(?(value_leftquote)(?P<value_rightquote>(\\?[\"']){1,3}))
+  filter_type: GeneralKeyword
+  use_ml: true
+  required_substrings:
+    - token
+    - oauth2-bearer
+  min_line_len: 12
+  target:
+    - code
+
+- name: CMD Secret
+  severity: high
+  confidence: moderate
+  type: pattern
+  values:
+    - (^|\W|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<variable>-[A-Za-z_-]*(?i:secret)[A-Za-z_-]*)(\s|\\?[\"'],)\s*(?!-)(?P<value_leftquote>(\\?[\"']){1,3})?(pass:)?(?!file:|env:|fd:)(?P<value>(?(value_leftquote)[^\"'\\]|[^\s\"'\\]){4,4000})(?(value_leftquote)(?P<value_rightquote>(\\?[\"']){1,3}))
+  filter_type: GeneralKeyword
+  use_ml: true
+  required_substrings:
+    - secret
+  min_line_len: 12
+  target:
+    - code
+
+- name: URL Credentials
+  severity: high
+  confidence: moderate
+  type: pattern
+  values:
+    - (?P<value_leftquote>[\"'])?(?P<variable>[+0-9A-Za-z-]{2,80}://)([^\s\'"<>\[\]^~`{|}:/]{0,80}:){1,3}(?P<value>[^\s\'"<>\[\]^~`{|}@:/]{3,80})@[^\s\'"<>\[\]^~`{|}@:/]{1,800}\\{0,8}(?P<value_rightquote>[\"'])?
+  filter_type: UrlCredentialsGroup
+  use_ml: true
+  required_substrings:
+    - ://
+  min_line_len: 10
+  target:
+    - code
+
 - name: API
   severity: low
   confidence: moderate
@@ -1677,7 +1678,7 @@
   confidence: moderate
   type: keyword
   values:
-    - (?<!by)pass(?!ed|ing|ion|es|age|\s+[a-z]{3,80})|pw(d|\b)
+    - (?<!by)pass(?!e[dns]|ing|ion|age|\s+[a-z]{3,80})|pw(d|\b)
   filter_type: PasswordKeyword
   use_ml: true
   min_line_len: 10

credsweeper/scanner/scanner.py

@@ -19,6 +19,8 @@ from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 
+RULES_PATH = APP_PATH / "rules" / "config.yaml"
+
 
 class Scanner:
     """Advanced Credential Scanner base class.
@@ -66,11 +68,11 @@ class Scanner:
                 return True
         return False
 
-    def _set_rules_scanners(self, rule_path: Union[None, str, Path]) -> None:
+    def _set_rules_scanners(self, rules_path: Union[None, str, Path]) -> None:
         """Auxiliary method to fill rules, determine min_pattern_len and set scanners"""
-        if rule_path is None:
-            rule_path = APP_PATH / "rules" / "config.yaml"
-        rule_templates = Util.yaml_load(rule_path)
+        if rules_path is None:
+            rules_path = RULES_PATH
+        rule_templates = Util.yaml_load(rules_path)
         if rule_templates and isinstance(rule_templates, list):
             rule_names = set()
             for rule_template in rule_templates:
@@ -98,7 +100,7 @@ class Scanner:
                         logger.warning(f"Unknown rule type:{rule.rule_type}")
                 self.rules_scanners.append((rule, self.get_scanner(rule)))
         else:
-            raise RuntimeError(f"Wrong rules '{rule_templates}' were read from '{rule_path}'")
+            raise RuntimeError(f"Wrong rules '{rule_templates}' were read from '{rules_path}'")
 
     def _is_available(self, rule: Rule) -> bool:
         """separate the method to reduce complexity"""
@@ -153,8 +155,11 @@ class Scanner:
                 target_line_stripped_len >= self.min_keyword_len and (  #
                         '=' in target_line_stripped
                         or ':' in target_line_stripped
-                        or "#define" in target_line_stripped
-                        or "%define" in target_line_stripped
+                        or ("define" in target_line_stripped
+                            and ('(' in target_line_stripped and ',' in target_line_stripped
+                                 or "#define" in target_line_stripped
+                                 or "%define" in target_line_stripped)
+                            )
                         or "%global" in target_line_stripped
                         or "set" in target_line_stripped_lower
                         or "%3d" in target_line_stripped_lower

credsweeper/secret/config.json

@@ -12,18 +12,21 @@
             ".rpm",
             ".tar",
             ".war",
+            ".whl",
             ".xz",
             ".zip"
         ],
         "documents": [
-            ".xlsx",
+            ".doc",
             ".docx",
-            ".pptx",
-            ".xls",
             ".odp",
             ".ods",
             ".odt",
-            ".pdf"
+            ".pdf",
+            ".ppt",
+            ".pptx",
+            ".xls",
+            ".xlsx"
         ],
         "extension": [
             ".7z",
@@ -45,16 +48,23 @@
             ".info",
             ".jpeg",
             ".jpg",
+            ".lib",
             ".map",
             ".m4a",
             ".mat",
             ".mo",
+            ".mov",
             ".mp3",
             ".mp4",
+            ".mpg",
+            ".mkv",
             ".npy",
             ".npz",
             ".obj",
+            ".oga",
             ".ogg",
+            ".ogv",
+            ".ops",
             ".pak",
             ".png",
             ".psd",
@@ -71,8 +81,10 @@
             ".so",
             ".sum",
             ".svg",
+            ".swf",
             ".tif",
             ".tiff",
+            ".tlb",
             ".ttf",
             ".vcxproj",
             ".vdproj",
@@ -81,6 +93,7 @@
             ".webp",
             ".wma",
             ".woff",
+            ".woff2",
             ".yuv"
         ],
         "path": [
@@ -164,7 +177,7 @@
         "tizen"
     ],
     "check_for_literals": true,
-    "max_password_value_length": 31,
+    "max_password_value_length": 64,
     "max_url_cred_value_length": 80,
     "line_data_output": [
         "line",

credsweeper/utils/util.py

@@ -141,15 +141,6 @@ class Util:
         min_entropy = Util.get_min_data_entropy(data_len)
         return entropy < min_entropy
 
-    @staticmethod
-    def is_known(data: Union[bytes, bytearray]) -> bool:
-        """Returns True if any known binary format is found to prevent extra scan a file without an extension."""
-        if isinstance(data, (bytes, bytearray)) and data.startswith(b"\x7f\x45\x4c\x46") and 127 <= len(data):
-            # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format
-            # minimal ELF is 127 bytes https://github.com/tchajed/minimal-elf
-            return True
-        return False
-
     @staticmethod
     def is_binary(data: Union[bytes, bytearray]) -> bool:
         """
@@ -218,13 +209,12 @@ class Util:
             try:
                 if binary_suggest and LATIN_1 == encoding and (Util.is_binary(content) or not Util.is_latin1(content)):
                     # LATIN_1 may convert data (bytes in range 0x80:0xFF are transformed)
-                    # so skip this encoding when checking binaries
-                    logger.warning("Binary file detected %s", repr(content[:8]))
                     break
-                text = content.decode(encoding, errors="strict")
-                if content != text.encode(encoding, errors="strict"):
+                _text = content.decode(encoding=encoding, errors="strict")
+                if content != _text.encode(encoding=encoding, errors="strict"):
                     # the check helps to detect a real encoding
                     raise UnicodeError
+                text = _text
                 break
             except UnicodeError:
                 binary_suggest = True
@@ -233,6 +223,11 @@ class Util:
                 logger.error(f"Unexpected Error: Can't read content as {encoding}. Error message: {exc}")
         return text
 
+    @staticmethod
+    def split_text(text: str) -> List[str]:
+        """Splits a text into lines, handling all common line endings (e.g., LF, CRLF, CR)."""
+        return text.replace("\r\n", '\n').replace('\r', '\n').split('\n')
+
     @staticmethod
     def decode_bytes(content: bytes, encodings: Optional[List[str]] = None) -> List[str]:
         """Decode content using different encodings.
@@ -251,7 +246,7 @@ class Util:
 
         """
         if text := Util.decode_text(content, encodings):
-            lines = text.replace('\r\n', '\n').replace('\r', '\n').split('\n')
+            lines = Util.split_text(text)
         else:
             lines = []
         return lines
@@ -355,13 +350,20 @@ class Util:
             return True
         return False
 
-    @classmethod
-    def is_sqlite3(cls, data):
+    @staticmethod
+    def is_sqlite3(data: Union[bytes, bytearray]):
         """According https://en.wikipedia.org/wiki/List_of_file_signatures - SQLite Database"""
         if isinstance(data, (bytes, bytearray)) and data.startswith(b"SQLite format 3\0"):
             return True
         return False
 
+    @staticmethod
+    def is_rtf(data: Union[bytes, bytearray]):
+        """According https://en.wikipedia.org/wiki/List_of_file_signatures - Rich Text Format"""
+        if isinstance(data, (bytes, bytearray)) and data.startswith(b"{\\rtf1") and data.endswith(b"}"):
+            return True
+        return False
+
     @staticmethod
     def is_asn1(data: Union[bytes, bytearray]) -> int:
         """Only sequence type 0x30 and size correctness are checked
@@ -575,6 +577,7 @@ class Util:
         """decode text to bytes with / without padding detect and urlsafe symbols"""
         value = text.translate(Util.WHITESPACE_TRANS_TABLE)
         if padding_safe:
+            value = value.rstrip('=')  # python 3.10 workaround
             pad_num = 0x3 & len(value)
             if pad_num:
                 value += '=' * (4 - pad_num)