credsweeper 1.10.8__tar.gz → 1.11.0__tar.gz

{credsweeper-1.10.8 → credsweeper-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.10.8
+Version: 1.11.0
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues

{credsweeper-1.10.8 → credsweeper-1.11.0}/credsweeper/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = "1.10.8"
+__version__ = "1.11.0"

{credsweeper-1.10.8 → credsweeper-1.11.0}/credsweeper/common/morpheme_checklist.txt

@@ -960,6 +960,7 @@ nish
 nism
 node
 non
+nope
 norm
 not
 nsive
@@ -1529,6 +1530,7 @@ warn
 watch
 wave
 way
+weak
 web
 week
 weight

{credsweeper-1.10.8 → credsweeper-1.11.0}/credsweeper/ml_model/features/__init__.py

@@ -6,7 +6,9 @@ from credsweeper.ml_model.features.length_of_attribute import LengthOfAttribute
 from credsweeper.ml_model.features.morpheme_dense import MorphemeDense
 from credsweeper.ml_model.features.rule_name import RuleName
 from credsweeper.ml_model.features.search_in_attribute import SearchInAttribute
-from credsweeper.ml_model.features.word_in_line import WordInLine
 from credsweeper.ml_model.features.word_in_path import WordInPath
+from credsweeper.ml_model.features.word_in_postamble import WordInPostamble
+from credsweeper.ml_model.features.word_in_preamble import WordInPreamble
+from credsweeper.ml_model.features.word_in_transition import WordInTransition
 from credsweeper.ml_model.features.word_in_value import WordInValue
 from credsweeper.ml_model.features.word_in_variable import WordInVariable

credsweeper-1.11.0/credsweeper/ml_model/features/word_in_postamble.py

@@ -0,0 +1,32 @@
+from typing import List
+
+import numpy as np
+
+from credsweeper.common.constants import ML_HUNK
+from credsweeper.credentials import Candidate
+from credsweeper.ml_model.features.word_in import WordIn
+
+
+class WordInPostamble(WordIn):
+    """Feature is true if line contains at least one word from predefined list."""
+
+    def __init__(self, words: List[str]) -> None:
+        """Feature returns array of matching words
+
+        Args:
+            words: list of predefined words - MUST BE IN LOWER CASE
+
+        """
+        super().__init__(words)
+
+    def extract(self, candidate: Candidate) -> np.ndarray:
+        """Returns true if any words in a part of line after value"""
+        postamble_end = len(candidate.line_data_list[0].line) \
+            if len(candidate.line_data_list[0].line) < candidate.line_data_list[0].value_end + ML_HUNK \
+            else candidate.line_data_list[0].value_end + ML_HUNK
+        postamble = candidate.line_data_list[0].line[candidate.line_data_list[0].value_end:postamble_end].strip()
+
+        if postamble:
+            return self.word_in_str(postamble.lower())
+        else:
+            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper-1.11.0/credsweeper/ml_model/features/word_in_preamble.py

@@ -0,0 +1,37 @@
+from typing import List
+
+import numpy as np
+
+from credsweeper.common.constants import ML_HUNK
+from credsweeper.credentials import Candidate
+from credsweeper.ml_model.features.word_in import WordIn
+
+
+class WordInPreamble(WordIn):
+    """Feature is true if line contains at least one word from predefined list."""
+
+    def __init__(self, words: List[str]) -> None:
+        """Feature returns array of matching words
+
+        Args:
+            words: list of predefined words - MUST BE IN LOWER CASE
+
+        """
+        super().__init__(words)
+
+    def extract(self, candidate: Candidate) -> np.ndarray:
+        """Returns true if any words in line before variable or value"""
+        if 0 <= candidate.line_data_list[0].variable_start:
+            preamble_start = 0 if ML_HUNK >= candidate.line_data_list[0].variable_start \
+                else candidate.line_data_list[0].variable_start - ML_HUNK
+            preamble = candidate.line_data_list[0].line[preamble_start:candidate.line_data_list[0].
+                                                        variable_start].strip()
+        else:
+            preamble_start = 0 if ML_HUNK >= candidate.line_data_list[0].value_start \
+                else candidate.line_data_list[0].value_start - ML_HUNK
+            preamble = candidate.line_data_list[0].line[preamble_start:candidate.line_data_list[0].value_start].strip()
+
+        if preamble:
+            return self.word_in_str(preamble.lower())
+        else:
+            return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

credsweeper-1.10.8/credsweeper/ml_model/features/word_in_line.py → credsweeper-1.11.0/credsweeper/ml_model/features/word_in_transition.py

@@ -2,13 +2,11 @@ from typing import List
 
 import numpy as np
 
-from credsweeper.common.constants import CHUNK_SIZE
 from credsweeper.credentials import Candidate
 from credsweeper.ml_model.features.word_in import WordIn
-from credsweeper.utils import Util
 
 
-class WordInLine(WordIn):
+class WordInTransition(WordIn):
     """Feature is true if line contains at least one word from predefined list."""
 
     def __init__(self, words: List[str]) -> None:
@@ -21,9 +19,14 @@ class WordInLine(WordIn):
         super().__init__(words)
 
     def extract(self, candidate: Candidate) -> np.ndarray:
-        """Returns true if any words in first line"""
-        subtext = Util.subtext(candidate.line_data_list[0].line, candidate.line_data_list[0].value_start, CHUNK_SIZE)
-        if subtext:
-            return self.word_in_str(subtext.lower())
+        """Returns true if any words between variable and value"""
+        if 0 <= candidate.line_data_list[0].variable_end < candidate.line_data_list[0].value_start:
+            transition = candidate.line_data_list[0].line[candidate.line_data_list[0].variable_end:candidate.
+                                                          line_data_list[0].value_start].strip()
+        else:
+            transition = ''
+
+        if transition:
+            return self.word_in_str(transition.lower())
         else:
             return np.array([np.zeros(shape=[self.dimension], dtype=np.int8)])

{credsweeper-1.10.8 → credsweeper-1.11.0}/credsweeper/ml_model/ml_config.json

@@ -70,6 +70,38 @@
                 "attribute": "value"
             }
         },
+        {
+            "type": "SearchInAttribute",
+            "comment": "camelStyle naming detection",
+            "kwargs": {
+                "pattern": "^[a-z][a-z]{1,16}[0-9]*([A-Z]([a-z]{1,16}[0-9]*|[0-9]{1,16})){1,8}$",
+                "attribute": "value"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "PascalStyle naming detection",
+            "kwargs": {
+                "pattern": "^([A-Z]([a-z]{1,16}[0-9]*|[0-9]{1,16})){1,8}$",
+                "attribute": "value"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "UPPERCASE naming detection",
+            "kwargs": {
+                "pattern": "^(_+[0-9]{1,16}|_*[A-Z]{1,16}[0-9]*)(_+([0-9]{1,16}|[A-Z]{1,16}[0-9]*)){1,8}_*$",
+                "attribute": "value"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "lowercase naming detection",
+            "kwargs": {
+                "pattern": "^(_+[0-9]{1,16}|_*[a-z]{1,16}[0-9]*)(_+([0-9]{1,16}|[a-z]{1,16}[0-9]*)){1,8}_*$",
+                "attribute": "value"
+            }
+        },
         {
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedPatternCheck",
@@ -82,7 +114,7 @@
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedNameCheck",
             "kwargs": {
-                "pattern": "(?i:pub(lic)?_?key)",
+                "pattern": "(?i:(filters?|pub(lic)?)_?key)",
                 "attribute": "variable"
             }
         },
@@ -90,7 +122,15 @@
             "type": "SearchInAttribute",
             "comment": "VariableNotAllowedNameCheck",
             "kwargs": {
-                "pattern": "(?i:_?id$|name$|type$)",
+                "pattern": "(?i:(id|size|name|type|manager)$)",
+                "attribute": "variable"
+            }
+        },
+        {
+            "type": "SearchInAttribute",
+            "comment": "PWD invocation",
+            "kwargs": {
+                "pattern": "(?i:(^\\$pwd$)|(^\\$\\{#?pwd[^}]*\\}$)|(^\\$\\(pwd\\)$)|(^`pwd`$))",
                 "attribute": "variable"
             }
         },
@@ -98,44 +138,55 @@
             "type": "WordInVariable",
             "kwargs": {
                 "words": [
-                    "/",
                     " ",
+                    "/",
                     "_at",
                     "_id",
-                    "obj",
+                    "_len",
+                    "access",
+                    "cache",
+                    "client",
+                    "control",
+                    "encrypted",
+                    "example",
+                    "expire",
+                    "fake",
                     "file",
-                    "path",
+                    "filter",
+                    "fingerprint",
                     "hash",
+                    "key",
+                    "label",
+                    "length",
+                    "manager",
+                    "mock",
                     "name",
+                    "native",
+                    "obj",
+                    "option",
+                    "p/w",
+                    "parameter",
+                    "pass",
+                    "path",
+                    "project",
+                    "public",
+                    "pw",
+                    "secret",
+                    "size",
+                    "space",
+                    "status",
+                    "sword",
+                    "temp",
                     "test",
+                    "thumbprint",
                     "time",
-                    "temp",
+                    "timestamp",
+                    "title",
+                    "token",
                     "type",
-                    "mock",
-                    "size",
                     "uniq",
-                    "fake",
-                    "view",
-                    "cache",
                     "valid",
-                    "label",
-                    "title",
-                    "access",
-                    "space",
-                    "filter",
-                    "native",
-                    "status",
-                    "expire",
-                    "client",
-                    "option",
-                    "public",
-                    "project",
-                    "control",
-                    "parameter",
-                    "encrypted",
-                    "timestamp",
-                    "thumbprint",
-                    "fingerprint"
+                    "view"
                 ]
             }
         },
@@ -144,76 +195,154 @@
             "kwargs": {
                 "words": [
                     " ",
+                    "$(",
+                    "${",
                     "(",
-                    "[",
-                    ".",
                     "->",
-                    "${",
-                    "$(",
+                    ".",
                     "...",
-                    "foo",
-                    "bar",
                     "123",
+                    "<",
+                    ">",
+                    "[",
+                    "_id",
                     "abc",
-                    "xyz",
-                    "xxx",
-                    "pwd",
-                    "passwd",
-                    "pswd",
-                    "psswd",
+                    "allow",
+                    "bar",
+                    "disable",
+                    "changeme",
+                    "example",
+                    "fake",
+                    "file",
+                    "foo",
+                    "min",
+                    "mock",
+                    "my",
+                    "nil",
                     "pass",
+                    "passwd",
                     "password",
-                    "pasword",
-                    "null",
-                    "nil",
-                    "undefined",
-                    "none",
-                    "true",
-                    "false",
-                    "example",
+                    "pswd",
                     "public",
-                    "mock",
-                    "fake",
+                    "pwd",
                     "test",
-                    "allow",
-                    "my",
-                    "file",
-                    "id"
+                    "xxx",
+                    "xyz"
                 ]
             }
         },
         {
-            "type": "WordInLine",
+            "type": "WordInPreamble",
             "kwargs": {
                 "words": [
+                    "$",
+                    "%2",
+                    "%3",
+                    "&",
+                    "&amp;",
                     "(",
-                    "[",
+                    "->",
                     ".",
-                    "$",
                     "://",
+                    "?",
                     "@",
-                    "pwd",
-                    "passwd",
-                    "pswd",
-                    "psswd",
+                    "[",
+                    "approval",
+                    "assert",
+                    "case",
+                    "circle",
+                    "equal",
+                    "example",
+                    "expect",
+                    "false",
+                    "height",
+                    "image",
+                    "line",
+                    "media",
+                    "nil",
+                    "none",
+                    "null",
                     "pass",
                     "password",
-                    "pasword",
-                    "->",
-                    "null",
-                    "nil",
+                    "path",
+                    "pwd",
+                    "sqa",
+                    "test",
+                    "true",
                     "undefined",
-                    "none",
                     "unit",
+                    "width"
+                ]
+            }
+        },
+        {
+            "type": "WordInTransition",
+            "kwargs": {
+                "words": [
+                    "%2",
+                    "%3",
+                    "&",
+                    "(",
+                    "->",
+                    ".",
+                    "?",
+                    "@",
+                    "[",
+                    "bearer",
+                    "equal",
+                    "example",
+                    "expect",
+                    "line",
+                    "media",
+                    "pass",
+                    "password",
+                    "path",
                     "test",
-                    "approval",
-                    "case",
-                    "true",
-                    "false",
+                    "unit"
+                ]
+            }
+        },
+        {
+            "type": "WordInPostamble",
+            "kwargs": {
+                "words": [
+                    "$",
+                    "%2",
+                    "%3",
+                    "&",
+                    "&amp;",
+                    "(",
+                    "->",
+                    ".",
+                    "://",
+                    "?",
+                    "@",
+                    "[",
                     "assert",
+                    "case",
+                    "circle",
                     "equal",
                     "example",
-                    "expect"
+                    "expect",
+                    "false",
+                    "height",
+                    "image",
+                    "line",
+                    "media",
+                    "nil",
+                    "none",
+                    "null",
+                    "pass",
+                    "passwd",
+                    "password",
+                    "path",
+                    "pwd",
+                    "sqa",
+                    "test",
+                    "true",
+                    "undefined",
+                    "unit",
+                    "width"
                 ]
             }
         },
@@ -221,20 +350,22 @@
             "type": "WordInPath",
             "kwargs": {
                 "words": [
-                    "/test",
-                    "/config",
-                    "/src/",
-                    "/record",
-                    "/usr/local/lib/python",
+                    "/conf",
                     "/dist-packages/",
+                    "/example",
+                    "/record",
+                    "/script",
                     "/site-packages/",
-                    "/example"
+                    "/src/",
+                    "/test",
+                    "/tool",
+                    "/usr/local/lib/python",
+                    "/assets/"
                 ]
             }
         },
         {
-            "type": "MorphemeDense",
-            "kwargs": {}
+            "type": "MorphemeDense"
         },
         {
             "type": "HasHtmlTag"
@@ -255,6 +386,7 @@
                     ".bat",
                     ".bats",
                     ".bazel",
+                    ".bin",
                     ".build",
                     ".bundle",
                     ".bzl",
@@ -337,6 +469,7 @@
                     ".nix",
                     ".nolint",
                     ".odd",
+                    ".onnx",
                     ".oracle",
                     ".pan",
                     ".patch",
@@ -396,6 +529,7 @@
                     ".ts",
                     ".tsx",
                     ".txt",
+                    ".var",
                     ".vue",
                     ".xaml",
                     ".xib",