credsweeper 1.10.8__tar.gz → 1.11.1__tar.gz

{credsweeper-1.10.8 → credsweeper-1.11.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.10.8
+Version: 1.11.1
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues

{credsweeper-1.10.8 → credsweeper-1.11.1}/credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.10.8"
+__version__ = "1.11.1"

{credsweeper-1.10.8 → credsweeper-1.11.1}/credsweeper/common/morpheme_checklist.txt

@@ -960,6 +960,7 @@ nish
 nism
 node
 non
+nope
 norm
 not
 nsive
@@ -1529,6 +1530,7 @@ warn
 watch
 wave
 way
+weak
 web
 week
 weight

{credsweeper-1.10.8 → credsweeper-1.11.1}/credsweeper/deep_scanner/deep_scanner.py

@@ -28,6 +28,7 @@ from .pdf_scanner import PdfScanner
 from .pkcs12_scanner import Pkcs12Scanner
 from .pptx_scanner import PptxScanner
 from .tar_scanner import TarScanner
+from .tmx_scanner import TmxScanner
 from .xlsx_scanner import XlsxScanner
 from .xml_scanner import XmlScanner
 from .zip_scanner import ZipScanner
@@ -126,6 +127,10 @@ class DeepScanner(
                 deep_scanners.append(MxfileScanner)
                 deep_scanners.append(XmlScanner)
                 fallback_scanners.append(ByteScanner)
+            elif Util.is_tmx(data):
+                deep_scanners.append(TmxScanner)
+                fallback_scanners.append(XmlScanner)
+                fallback_scanners.append(ByteScanner)
             else:
                 deep_scanners.append(XmlScanner)
                 fallback_scanners.append(ByteScanner)

credsweeper-1.11.1/credsweeper/deep_scanner/tmx_scanner.py

@@ -0,0 +1,45 @@
+import logging
+from abc import ABC
+from typing import List, Optional
+
+from lxml import etree
+
+from credsweeper.common.constants import MIN_DATA_LEN
+from credsweeper.credentials import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.string_content_provider import StringContentProvider
+from credsweeper.utils import Util
+
+logger = logging.getLogger(__name__)
+
+
+class TmxScanner(AbstractScanner, ABC):
+    """Realises tmX files scanning for values only. Image tags are skipped."""
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Tries to represent data as xml text and scan as text lines"""
+        try:
+            lines = []
+            # the format is always in single line xlm, so line numbers are not actual
+            tree = etree.fromstring(data_provider.data)
+            for element in tree.iter():
+                tag = Util.extract_element_data(element, "tag")
+                if "Image" in tag:
+                    continue
+                text = Util.extract_element_data(element, "text")
+                if MIN_DATA_LEN > len(text):
+                    continue
+                lines.append(text)
+            tmx_data_provider = StringContentProvider(lines=lines,
+                                                      file_path=data_provider.file_path,
+                                                      file_type=data_provider.file_type,
+                                                      info=f"{data_provider.info}|TMX")
+            return self.scanner.scan(tmx_data_provider)
+        except Exception as exc:
+            logger.warning("Cannot processed tmX file %s %s", str(data_provider.file_path), str(exc))
+        return None

{credsweeper-1.10.8 → credsweeper-1.11.1}/credsweeper/ml_model/features/__init__.py

@@ -6,7 +6,9 @@ from credsweeper.ml_model.features.length_of_attribute import LengthOfAttribute
 from credsweeper.ml_model.features.morpheme_dense import MorphemeDense
 from credsweeper.ml_model.features.rule_name import RuleName
 from credsweeper.ml_model.features.search_in_attribute import SearchInAttribute
-from credsweeper.ml_model.features.word_in_line import WordInLine
 from credsweeper.ml_model.features.word_in_path import WordInPath
+from credsweeper.ml_model.features.word_in_postamble import WordInPostamble
+from credsweeper.ml_model.features.word_in_preamble import WordInPreamble
+from credsweeper.ml_model.features.word_in_transition import WordInTransition
 from credsweeper.ml_model.features.word_in_value import WordInValue
 from credsweeper.ml_model.features.word_in_variable import WordInVariable

credsweeper-1.11.1/credsweeper/ml_model/features/word_in_postamble.py

@@ -0,0 +1,32 @@
+from typing import List
+
+import numpy as np
+
+from credsweeper.common.constants import ML_HUNK
+from credsweeper.credentials import Candidate
+from credsweeper.ml_model.features.word_in import WordIn
+
+
+class WordInPostamble(WordIn):
+    """Feature is true if line contains at least one word from predefined list."""
+
+    def __init__(self, words: List[str]) -> None:
+        """Feature returns array of matching words
+
+        Args:
+            words: list of predefined words - MUST BE IN LOWER CASE
+
+        """
+        super().__init__(words)
+
+    def extract(self, candidate: Candidate) -> np.ndarray:
+        """Returns true if any words in a part of line after value"""
+        postamble_end = len(candidate.line_data_list[0].line) \
+            if len(candidate.line_data_list[0].line) < candidate.line_data_list[0].value_end + ML_HUNK \
+            else candidate.line_data_list[0].value_end + ML_HUNK
+        postamble = candidate.line_data_list[0].line[candidate.line_data_list[0].value_end:postamble_end].strip()
+
+        if postamble:
+            return self.word_in_str(postamble.lower())
+        else:
+            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper-1.11.1/credsweeper/ml_model/features/word_in_preamble.py

@@ -0,0 +1,37 @@
+from typing import List
+
+import numpy as np
+
+from credsweeper.common.constants import ML_HUNK
+from credsweeper.credentials import Candidate
+from credsweeper.ml_model.features.word_in import WordIn
+
+
+class WordInPreamble(WordIn):
+    """Feature is true if line contains at least one word from predefined list."""
+
+    def __init__(self, words: List[str]) -> None:
+        """Feature returns array of matching words
+
+        Args:
+            words: list of predefined words - MUST BE IN LOWER CASE
+
+        """
+        super().__init__(words)
+
+    def extract(self, candidate: Candidate) -> np.ndarray:
+        """Returns true if any words in line before variable or value"""
+        if 0 <= candidate.line_data_list[0].variable_start:
+            preamble_start = 0 if ML_HUNK >= candidate.line_data_list[0].variable_start \
+                else candidate.line_data_list[0].variable_start - ML_HUNK
+            preamble = candidate.line_data_list[0].line[preamble_start:candidate.line_data_list[0].
+                                                        variable_start].strip()
+        else:
+            preamble_start = 0 if ML_HUNK >= candidate.line_data_list[0].value_start \
+                else candidate.line_data_list[0].value_start - ML_HUNK
+            preamble = candidate.line_data_list[0].line[preamble_start:candidate.line_data_list[0].value_start].strip()
+
+        if preamble:
+            return self.word_in_str(preamble.lower())
+        else:
+            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper-1.10.8/credsweeper/ml_model/features/word_in_line.py → credsweeper-1.11.1/credsweeper/ml_model/features/word_in_transition.py

@@ -2,13 +2,11 @@ from typing import List
 
 import numpy as np
 
-from credsweeper.common.constants import CHUNK_SIZE
 from credsweeper.credentials import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
-from credsweeper.utils import Util
 
 
-class WordInLine(WordIn):
+class WordInTransition(WordIn):
     """Feature is true if line contains at least one word from predefined list."""
 
     def __init__(self, words: List[str]) -> None:
@@ -21,9 +19,14 @@ class WordInLine(WordIn):
         super().__init__(words)
 
     def extract(self, candidate: Candidate) -> np.ndarray:
-        """Returns true if any words in first line"""
-        subtext = Util.subtext(candidate.line_data_list[0].line, candidate.line_data_list[0].value_start, CHUNK_SIZE)
-        if subtext:
-            return self.word_in_str(subtext.lower())
+        """Returns true if any words between variable and value"""
+        if 0 <= candidate.line_data_list[0].variable_end < candidate.line_data_list[0].value_start:
+            transition = candidate.line_data_list[0].line[candidate.line_data_list[0].variable_end:candidate.
+                                                          line_data_list[0].value_start].strip()
+        else:
+            transition = ''
+
+        if transition:
+            return self.word_in_str(transition.lower())
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

{credsweeper-1.10.8 → credsweeper-1.11.1}/credsweeper/ml_model/ml_config.json

@@ -70,6 +70,38 @@
                 "attribute": "value"
             }
         },
+        {
+            "type": "SearchInAttribute",
+            "comment": "camelStyle naming detection",
+            "kwargs": {
+                "pattern": "^[a-z][a-z]{1,16}[0-9]*([A-Z]([a-z]{1,16}[0-9]*|[0-9]{1,16})){1,8}$",
+                "attribute": "value"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "PascalStyle naming detection",
+            "kwargs": {
+                "pattern": "^([A-Z]([a-z]{1,16}[0-9]*|[0-9]{1,16})){1,8}$",
+                "attribute": "value"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "UPPERCASE naming detection",
+            "kwargs": {
+                "pattern": "^(_+[0-9]{1,16}|_*[A-Z]{1,16}[0-9]*)(_+([0-9]{1,16}|[A-Z]{1,16}[0-9]*)){1,8}_*$",
+                "attribute": "value"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "lowercase naming detection",
+            "kwargs": {
+                "pattern": "^(_+[0-9]{1,16}|_*[a-z]{1,16}[0-9]*)(_+([0-9]{1,16}|[a-z]{1,16}[0-9]*)){1,8}_*$",
+                "attribute": "value"
+            }
+        },
         {
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedPatternCheck",
@@ -82,7 +114,7 @@
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedNameCheck",
             "kwargs": {
-                "pattern": "(?i:pub(lic)?_?key)",
+                "pattern": "(?i:(filters?|pub(lic)?)_?key)",
                 "attribute": "variable"
             }
         },
@@ -90,7 +122,15 @@
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedNameCheck",
             "kwargs": {
-                "pattern": "(?i:_?id$|name$|type$)",
+                "pattern": "(?i:(id|size|name|type|manager)$)",
+                "attribute": "variable"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "PWD invocation",
+            "kwargs": {
+                "pattern": "(?i:(^\\$pwd$)|(^\\$\\{#?pwd[^}]*\\}$)|(^\\$\\(pwd\\)$)|(^`pwd`$))",
                 "attribute": "variable"
             }
         },
@@ -98,44 +138,55 @@
             "type": "WordInVariable",
             "kwargs": {
                 "words": [
-                    "/",
                     " ",
+                    "/",
                     "_at",
                     "_id",
-                    "obj",
+                    "_len",
+                    "access",
+                    "cache",
+                    "client",
+                    "control",
+                    "encrypted",
+                    "example",
+                    "expire",
+                    "fake",
                     "file",
-                    "path",
+                    "filter",
+                    "fingerprint",
                     "hash",
+                    "key",
+                    "label",
+                    "length",
+                    "manager",
+                    "mock",
                     "name",
+                    "native",
+                    "obj",
+                    "option",
+                    "p/w",
+                    "parameter",
+                    "pass",
+                    "path",
+                    "project",
+                    "public",
+                    "pw",
+                    "secret",
+                    "size",
+                    "space",
+                    "status",
+                    "sword",
+                    "temp",
                     "test",
+                    "thumbprint",
                     "time",
-                    "temp",
+                    "timestamp",
+                    "title",
+                    "token",
                     "type",
-                    "mock",
-                    "size",
                     "uniq",
-                    "fake",
-                    "view",
-                    "cache",
                     "valid",
-                    "label",
-                    "title",
-                    "access",
-                    "space",
-                    "filter",
-                    "native",
-                    "status",
-                    "expire",
-                    "client",
-                    "option",
-                    "public",
-                    "project",
-                    "control",
-                    "parameter",
-                    "encrypted",
-                    "timestamp",
-                    "thumbprint",
-                    "fingerprint"
+                    "view"
                 ]
             }
         },
@@ -144,76 +195,154 @@
             "kwargs": {
                 "words": [
                     " ",
+                    "$(",
+                    "${",
                     "(",
-                    "[",
-                    ".",
                     "->",
-                    "${",
-                    "$(",
+                    ".",
                     "...",
-                    "foo",
-                    "bar",
                     "123",
+                    "<",
+                    ">",
+                    "[",
+                    "_id",
                     "abc",
-                    "xyz",
-                    "xxx",
-                    "pwd",
-                    "passwd",
-                    "pswd",
-                    "psswd",
+                    "allow",
+                    "bar",
+                    "disable",
+                    "changeme",
+                    "example",
+                    "fake",
+                    "file",
+                    "foo",
+                    "min",
+                    "mock",
+                    "my",
+                    "nil",
                     "pass",
+                    "passwd",
                     "password",
-                    "pasword",
-                    "null",
-                    "nil",
-                    "undefined",
-                    "none",
-                    "true",
-                    "false",
-                    "example",
+                    "pswd",
                     "public",
-                    "mock",
-                    "fake",
+                    "pwd",
                     "test",
-                    "allow",
-                    "my",
-                    "file",
-                    "id"
+                    "xxx",
+                    "xyz"
                 ]
             }
         },
         {
-            "type": "WordInLine",
+            "type": "WordInPreamble",
             "kwargs": {
                 "words": [
+                    "$",
+                    "%2",
+                    "%3",
+                    "&",
+                    "&amp;",
                     "(",
-                    "[",
+                    "->",
                     ".",
-                    "$",
                     "://",
+                    "?",
                     "@",
-                    "pwd",
-                    "passwd",
-                    "pswd",
-                    "psswd",
+                    "[",
+                    "approval",
+                    "assert",
+                    "case",
+                    "circle",
+                    "equal",
+                    "example",
+                    "expect",
+                    "false",
+                    "height",
+                    "image",
+                    "line",
+                    "media",
+                    "nil",
+                    "none",
+                    "null",
                     "pass",
                     "password",
-                    "pasword",
-                    "->",
-                    "null",
-                    "nil",
+                    "path",
+                    "pwd",
+                    "sqa",
+                    "test",
+                    "true",
                     "undefined",
-                    "none",
                     "unit",
+                    "width"
+                ]
+            }
+        },
+        {
+            "type": "WordInTransition",
+            "kwargs": {
+                "words": [
+                    "%2",
+                    "%3",
+                    "&",
+                    "(",
+                    "->",
+                    ".",
+                    "?",
+                    "@",
+                    "[",
+                    "bearer",
+                    "equal",
+                    "example",
+                    "expect",
+                    "line",
+                    "media",
+                    "pass",
+                    "password",
+                    "path",
                     "test",
-                    "approval",
-                    "case",
-                    "true",
-                    "false",
+                    "unit"
+                ]
+            }
+        },
+        {
+            "type": "WordInPostamble",
+            "kwargs": {
+                "words": [
+                    "$",
+                    "%2",
+                    "%3",
+                    "&",
+                    "&amp;",
+                    "(",
+                    "->",
+                    ".",
+                    "://",
+                    "?",
+                    "@",
+                    "[",
                     "assert",
+                    "case",
+                    "circle",
                     "equal",
                     "example",
-                    "expect"
+                    "expect",
+                    "false",
+                    "height",
+                    "image",
+                    "line",
+                    "media",
+                    "nil",
+                    "none",
+                    "null",
+                    "pass",
+                    "passwd",
+                    "password",
+                    "path",
+                    "pwd",
+                    "sqa",
+                    "test",
+                    "true",
+                    "undefined",
+                    "unit",
+                    "width"
                 ]
             }
         },
@@ -221,20 +350,22 @@
             "type": "WordInPath",
             "kwargs": {
                 "words": [
-                    "/test",
-                    "/config",
-                    "/src/",
-                    "/record",
-                    "/usr/local/lib/python",
+                    "/conf",
                     "/dist-packages/",
+                    "/example",
+                    "/record",
+                    "/script",
                     "/site-packages/",
-                    "/example"
+                    "/src/",
+                    "/test",
+                    "/tool",
+                    "/usr/local/lib/python",
+                    "/assets/"
                 ]
             }
         },
         {
-            "type": "MorphemeDense",
-            "kwargs": {}
+            "type": "MorphemeDense"
         },
         {
             "type": "HasHtmlTag"
@@ -255,6 +386,7 @@
                     ".bat",
                     ".bats",
                     ".bazel",
+                    ".bin",
                     ".build",
                     ".bundle",
                     ".bzl",
@@ -337,6 +469,7 @@
                     ".nix",
                     ".nolint",
                     ".odd",
+                    ".onnx",
                     ".oracle",
                     ".pan",
                     ".patch",
@@ -396,6 +529,7 @@
                     ".ts",
                     ".tsx",
                     ".txt",
+                    ".var",
                     ".vue",
                     ".xaml",
                     ".xib",

{credsweeper-1.10.8 → credsweeper-1.11.1}/credsweeper/utils/util.py

@@ -517,6 +517,18 @@ class Util:
                 return True
         return False
 
+    @staticmethod
+    def is_tmx(data: Union[bytes, bytearray]) -> bool:
+        """Used to detect tm7,tm6,etc. (ThreadModeling) format."""
+        if isinstance(data, (bytes, bytearray)):
+            for opening_tag, closing_tag in [(b"<ThreatModel", b"</ThreatModel>"),
+                                             (b"<KnowledgeBase", b"</KnowledgeBase>")]:
+                opening_pos = data.find(opening_tag, 0, MAX_LINE_LENGTH)
+                if 0 <= opening_pos < data.find(closing_tag, opening_pos):
+                    # opening and closing tags were found - suppose it is an HTML
+                    return True
+        return False
+
     # A well-formed XML must start from < or a whitespace character
     XML_FIRST_BRACKET_PATTERN = re.compile(rb"^\s*<")
     XML_OPENING_TAG_PATTERN = re.compile(rb"<([0-9A-Za-z_]{1,256})")
@@ -583,14 +595,14 @@ class Util:
         line_nums = []
         tree = etree.fromstringlist(xml_lines)
         for element in tree.iter():
-            tag = Util._extract_element_data(element, "tag")
-            text = Util._extract_element_data(element, "text")
+            tag = Util.extract_element_data(element, "tag")
+            text = Util.extract_element_data(element, "text")
             lines.append(f"{tag} : {text}")
             line_nums.append(element.sourceline)
         return lines, line_nums
 
     @staticmethod
-    def _extract_element_data(element, attr) -> str:
+    def extract_element_data(element: Any, attr: str) -> str:
         """Extract xml element data to string.
 
         Try to extract the xml data and strip() the string.
@@ -605,7 +617,7 @@ class Util:
         """
         element_attr: Any = getattr(element, attr)
         if element_attr is None or not isinstance(element_attr, str):
-            return ""
+            return ''
         return str(element_attr).strip()
 
     @staticmethod