credsweeper 1.11.0__tar.gz → 1.11.2__tar.gz

{credsweeper-1.11.0 → credsweeper-1.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.11.0
+Version: 1.11.2
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues

{credsweeper-1.11.0 → credsweeper-1.11.2}/credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.11.0"
+__version__ = "1.11.2"

{credsweeper-1.11.0 → credsweeper-1.11.2}/credsweeper/common/keyword_pattern.py

@@ -8,20 +8,20 @@ class KeywordPattern:
                r"(?P<keyword>"
     # there will be inserted a keyword
     key_right = r")" \
-                r"[^%:='\"`<>{?!&]*" \
+                r"[^%:='\"`<>{?!&;\n]*" \
                 r")" \
                 r"(&(quot|apos);|%[0-9a-f]{2}|[`'\"])*" \
                 r")"  # <variable>
     separator = r"(\s|\\{1,8}[tnr])*\]?(\s|\\{1,8}[tnr])*" \
-                r"(?P<separator>:(\s[a-z]{3,9}[?]?\s)?=|:|=(>|&gt;|\\u0026gt;)|!==|!=|===|==|=|%3d)" \
+                r"(?P<separator>:(\s[a-z]{3,9}[?]?\s)?=|:(?!:)|=(>|&gt;|(\\\\*u00|%)26gt;)|!==|!=|===|==|=|%3d)" \
                 r"(\s|\\{1,8}[tnr])*"
     # might be curly, square or parenthesis with words before
     wrap = r"(?P<wrap>(" \
-           r"(new(\s|\\{1,8}[tnr]){1,8})?" \
-           r"([0-9a-z_.]|-(>|(&|\\\\*u0026)gt;))*" \
+           r"(new(\s|\\{1,8}[tnr]|byte|char|string|\[\]){1,8})?" \
+           r"([0-9a-z_.]|::|-(>|&gt;))*" \
            r"[\[\(\{]" \
            r"(\s|\\{1,8}[tnr])*" \
-           r"([0-9a-z_]{1,32}=)?" \
+           r"([0-9a-z_]{1,32}[:=]\s*)?" \
            r"){1,8})?"
     string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[`'\"])))?"
     left_quote = r"(?P<value_leftquote>((?P<esq>\\{1,8})?([`'\"]|&(quot|apos);)){1,4}))?"
@@ -39,14 +39,22 @@ class KeywordPattern:
             r"(?P<url_esc>%[0-9a-f]{2})" \
             r"|" \
             r"(?(url_esc)[^\s`'\",;\\&]|[^\s`'\",;\\])" \
-            r")){3,8000}" \
-            r"|(\{[^}]{3,8000}\})" \
-            r"|(<[^>]{3,8000}>)" \
+            r")"\
+            r"){4,8000}" \
+            r"|" \
+            r"(<[^>]{4,8000}>)" \
+            r"|" \
+            r"(\$?\({1,3}[^)]{4,8000}\){1,3})" \
+            r"|" \
+            r"(\$?\{{1,3}[^}]{4,8000}\}{1,3})" \
+            r"|" \
+            r"(?(wrap)(?(value_leftquote)(?!\\(?P=value_leftquote))|[^\]\)\}]){16,8000})"\
             r")"  # <value>
     right_quote = r"(?(value_leftquote)" \
                   r"(?P<value_rightquote>(?<!\\)(?P=value_leftquote)|\\$|(?<=[0-9a-z+_/-])$)" \
                   r"|" \
-                  r"(?(wrap)[\]\)\},;]))"
+                  r"(?(wrap)(\]|\)|\}|,|;|\\|$))" \
+                  r")"
 
     @classmethod
     def get_keyword_pattern(cls, keyword: str) -> re.Pattern:

{credsweeper-1.11.0 → credsweeper-1.11.2}/credsweeper/deep_scanner/deep_scanner.py

@@ -28,6 +28,7 @@ from .pdf_scanner import PdfScanner
 from .pkcs12_scanner import Pkcs12Scanner
 from .pptx_scanner import PptxScanner
 from .tar_scanner import TarScanner
+from .tmx_scanner import TmxScanner
 from .xlsx_scanner import XlsxScanner
 from .xml_scanner import XmlScanner
 from .zip_scanner import ZipScanner
@@ -126,6 +127,10 @@ class DeepScanner(
                 deep_scanners.append(MxfileScanner)
                 deep_scanners.append(XmlScanner)
                 fallback_scanners.append(ByteScanner)
+            elif Util.is_tmx(data):
+                deep_scanners.append(TmxScanner)
+                fallback_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
             else:
                 deep_scanners.append(XmlScanner)
                 fallback_scanners.append(ByteScanner)

credsweeper-1.11.2/credsweeper/deep_scanner/tmx_scanner.py

@@ -0,0 +1,45 @@
+import logging
+from abc import ABC
+from typing import List, Optional
+
+from lxml import etree
+
+from credsweeper.common.constants import MIN_DATA_LEN
+from credsweeper.credentials import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.string_content_provider import StringContentProvider
+from credsweeper.utils import Util
+
+logger = logging.getLogger(__name__)
+
+
+class TmxScanner(AbstractScanner, ABC):
+    """Realises tmX files scanning for values only. Image tags are skipped."""
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Tries to represent data as xml text and scan as text lines"""
+        try:
+            lines = []
+            # the format is always in single line xlm, so line numbers are not actual
+            tree = etree.fromstring(data_provider.data)
+            for element in tree.iter():
+                tag = Util.extract_element_data(element, "tag")
+                if "Image" in tag:
+                    continue
+                text = Util.extract_element_data(element, "text")
+                if MIN_DATA_LEN > len(text):
+                    continue
+                lines.append(text)
+            tmx_data_provider = StringContentProvider(lines=lines,
+                                                      file_path=data_provider.file_path,
+                                                      file_type=data_provider.file_type,
+                                                      info=f"{data_provider.info}|TMX")
+            return self.scanner.scan(tmx_data_provider)
+        except Exception as exc:
+            logger.warning("Cannot processed tmX file %s %s", str(data_provider.file_path), str(exc))
+        return None

{credsweeper-1.11.0 → credsweeper-1.11.2}/credsweeper/filters/__init__.py

@@ -22,7 +22,6 @@ from credsweeper.filters.value_entropy_base32_check import ValueEntropyBase32Che
 from credsweeper.filters.value_entropy_base36_check import ValueEntropyBase36Check
 from credsweeper.filters.value_entropy_base64_check import ValueEntropyBase64Check
 from credsweeper.filters.value_file_path_check import ValueFilePathCheck
-from credsweeper.filters.value_first_word_check import ValueFirstWordCheck
 from credsweeper.filters.value_github_check import ValueGitHubCheck
 from credsweeper.filters.value_grafana_check import ValueGrafanaCheck
 from credsweeper.filters.value_grafana_service_check import ValueGrafanaServiceCheck

{credsweeper-1.11.0 → credsweeper-1.11.2}/credsweeper/filters/group/group.py

@@ -4,9 +4,9 @@ from typing import List
 from credsweeper.common.constants import GroupType
 from credsweeper.config import Config
 from credsweeper.filters import (Filter, LineSpecificKeyCheck, ValueAllowlistCheck, ValueArrayDictionaryCheck,
-                                 ValueBlocklistCheck, ValueCamelCaseCheck, ValueFilePathCheck, ValueFirstWordCheck,
-                                 ValueLastWordCheck, ValueMethodCheck, ValueNotAllowedPatternCheck, ValuePatternCheck,
-                                 ValueSimilarityCheck, ValueStringTypeCheck, ValueTokenCheck, ValueHexNumberCheck)
+                                 ValueBlocklistCheck, ValueCamelCaseCheck, ValueFilePathCheck, ValueLastWordCheck,
+                                 ValueMethodCheck, ValueNotAllowedPatternCheck, ValuePatternCheck, ValueSimilarityCheck,
+                                 ValueStringTypeCheck, ValueTokenCheck, ValueHexNumberCheck)
 
 
 class Group(ABC):
@@ -39,7 +39,6 @@ class Group(ABC):
             ValueBlocklistCheck(),
             ValueCamelCaseCheck(),
             ValueFilePathCheck(),
-            ValueFirstWordCheck(),
             ValueHexNumberCheck(),
             ValueLastWordCheck(),
             ValueMethodCheck(),

{credsweeper-1.11.0 → credsweeper-1.11.2}/credsweeper/filters/group/url_credentials_group.py

@@ -2,8 +2,8 @@ from credsweeper.common.constants import GroupType
 from credsweeper.config import Config
 from credsweeper.filters import (ValueAllowlistCheck, ValueArrayDictionaryCheck, ValueBlocklistCheck,
                                  ValueCamelCaseCheck, ValueDictionaryValueLengthCheck, ValueFilePathCheck,
-                                 ValueFirstWordCheck, ValueLastWordCheck, ValueMethodCheck, ValueNotAllowedPatternCheck,
-                                 ValuePatternCheck, ValueStringTypeCheck, ValueTokenCheck)
+                                 ValueLastWordCheck, ValueMethodCheck, ValueNotAllowedPatternCheck, ValuePatternCheck,
+                                 ValueStringTypeCheck, ValueTokenCheck)
 from credsweeper.filters.group import Group
 
 
@@ -23,7 +23,6 @@ class UrlCredentialsGroup(Group):
             ValueBlocklistCheck(),
             ValueCamelCaseCheck(),
             ValueFilePathCheck(),
-            ValueFirstWordCheck(),
             ValueLastWordCheck(),
             ValueMethodCheck(),
             ValueStringTypeCheck(config),

{credsweeper-1.11.0 → credsweeper-1.11.2}/credsweeper/filters/value_allowlist_check.py

@@ -25,6 +25,7 @@ class ValueAllowlistCheck(Filter):
 
     ALLOWED_QUOTED = [
         r"\$[a-z_]+[0-9a-z_]*([$\s]|$)",  #
+        r"\$\([^)]+\)",  #
         r".*\*\*\*",  #
     ]
 
@@ -33,6 +34,7 @@ class ValueAllowlistCheck(Filter):
     ALLOWED_UNQUOTED = [
         r"[~a-z0-9_]+((\.|->)[a-z0-9_]+)+\(.*$",  #
         r"\$[a-z_]+[0-9a-z_]*\b",  #
+        r"\$\([.0-9a-z_-]+",  #
         r".*\*\*\*\*\*",  #
     ]
 
@@ -52,14 +54,11 @@ class ValueAllowlistCheck(Filter):
             True, if need to filter candidate and False if left
 
         """
-
-        if self.ALLOWED_PATTERN.match(line_data.value):
-            return True
-        elif line_data.is_well_quoted_value:
-            if self.ALLOWED_QUOTED_PATTERN.match(line_data.value):
+        if line_data.is_well_quoted_value:
+            if self.ALLOWED_PATTERN.match(line_data.value) or self.ALLOWED_QUOTED_PATTERN.match(line_data.value):
                 return True
         else:
-            if self.ALLOWED_UNQUOTED_PATTERN.match(line_data.value):
+            value = line_data.wrap + line_data.value if line_data.wrap else line_data.value
+            if self.ALLOWED_PATTERN.match(value) or self.ALLOWED_UNQUOTED_PATTERN.match(value):
                 return True
-
         return False

{credsweeper-1.11.0 → credsweeper-1.11.2}/credsweeper/rules/config.yaml

@@ -448,7 +448,7 @@
   confidence: moderate
   type: keyword
   values:
-    - (?<!by)pass(?!ed|ing|es|\s+[a-z]{3,80})|pw(d|\b)
+    - (?<!by)pass(?!ed|ing|es|age|\s+[a-z]{3,80})|pw(d|\b)
   filter_type: PasswordKeyword
   use_ml: true
   min_line_len: 10
@@ -890,7 +890,7 @@
   confidence: moderate
   type: keyword
   values:
-    - nonce
+    - (?<!\\)nonce
   filter_type: GeneralKeyword
   use_ml: true
   min_line_len: 13

{credsweeper-1.11.0 → credsweeper-1.11.2}/credsweeper/utils/util.py

@@ -517,6 +517,18 @@ class Util:
                 return True
         return False
 
+    @staticmethod
+    def is_tmx(data: Union[bytes, bytearray]) -> bool:
+        """Used to detect tm7,tm6,etc. (ThreadModeling) format."""
+        if isinstance(data, (bytes, bytearray)):
+            for opening_tag, closing_tag in [(b"<ThreatModel", b"</ThreatModel>"),
+                                             (b"<KnowledgeBase", b"</KnowledgeBase>")]:
+                opening_pos = data.find(opening_tag, 0, MAX_LINE_LENGTH)
+                if 0 <= opening_pos < data.find(closing_tag, opening_pos):
+                    # opening and closing tags were found - suppose it is an HTML
+                    return True
+        return False
+
     # A well-formed XML must start from < or a whitespace character
     XML_FIRST_BRACKET_PATTERN = re.compile(rb"^\s*<")
     XML_OPENING_TAG_PATTERN = re.compile(rb"<([0-9A-Za-z_]{1,256})")
@@ -583,14 +595,14 @@ class Util:
         line_nums = []
         tree = etree.fromstringlist(xml_lines)
         for element in tree.iter():
-            tag = Util._extract_element_data(element, "tag")
-            text = Util._extract_element_data(element, "text")
+            tag = Util.extract_element_data(element, "tag")
+            text = Util.extract_element_data(element, "text")
             lines.append(f"{tag} : {text}")
             line_nums.append(element.sourceline)
         return lines, line_nums
 
     @staticmethod
-    def _extract_element_data(element, attr) -> str:
+    def extract_element_data(element: Any, attr: str) -> str:
         """Extract xml element data to string.
 
         Try to extract the xml data and strip() the string.
@@ -605,7 +617,7 @@ class Util:
         """
         element_attr: Any = getattr(element, attr)
         if element_attr is None or not isinstance(element_attr, str):
-            return ""
+            return ''
         return str(element_attr).strip()
 
     @staticmethod

credsweeper-1.11.0/credsweeper/filters/value_first_word_check.py

@@ -1,38 +0,0 @@
-import re
-
-from credsweeper.config import Config
-from credsweeper.credentials import LineData
-from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
-
-
-class ValueFirstWordCheck(Filter):
-    """Check that secret doesn't starts with special character."""
-
-    NOT_ALLOWED = [
-        r"\=", r"\{", r"\)", r"\<", r"\>", r"\#", r"\:", r"\\\\", r"\\/\\/", r"\_", r"\/\*", r"\%[deflspuvxz]"
-    ]
-    NOT_ALLOWED_PATTERN = re.compile(  #
-        f"^{Util.get_regex_combine_or(NOT_ALLOWED)}",  #
-        flags=re.IGNORECASE)
-
-    def __init__(self, config: Config = None) -> None:
-        pass
-
-    def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
-        """Run filter checks on received credential candidate data 'line_data'.
-
-        Args:
-            line_data: credential candidate data
-            target: multiline target from which line data was obtained
-
-        Return:
-            True, if need to filter candidate and False if left
-
-        """
-        if line_data.is_well_quoted_value:
-            return False
-        if self.NOT_ALLOWED_PATTERN.match(line_data.value):
-            return True
-        return False