credsweeper 1.13.0__tar.gz → 1.13.2__tar.gz

{credsweeper-1.13.0 → credsweeper-1.13.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.13.0
+Version: 1.13.2
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues

{credsweeper-1.13.0 → credsweeper-1.13.2}/credsweeper/__init__.py

@@ -24,4 +24,4 @@ __all__ = [
     "__version__"
 ]
 
-__version__ = "1.13.0"
+__version__ = "1.13.2"

{credsweeper-1.13.0 → credsweeper-1.13.2}/credsweeper/common/morpheme_checklist.txt

@@ -238,6 +238,7 @@ bless
 blic
 blish
 blob
+blood
 blue
 board
 bob
@@ -248,7 +249,7 @@ boost
 boot
 boss
 bot
-bound
+boun
 box
 branch
 break
@@ -613,6 +614,7 @@ fleet
 flick
 flix
 float
+flood
 floor
 fluent
 fluid
@@ -621,7 +623,7 @@ focus
 foo
 for
 fossil
-found
+foun
 fpga
 frame
 free
@@ -654,6 +656,7 @@ git
 given
 global
 gobble
+good
 google
 grab
 grace
@@ -709,6 +712,7 @@ home
 hook
 horizon
 host
+houn
 hours
 html
 http
@@ -862,7 +866,7 @@ local
 lock
 log
 long
-lookup
+look
 loop
 loose
 lost
@@ -955,6 +959,7 @@ ndow
 ned
 need
 neigh
+neo4j
 ner
 net
 neutr
@@ -999,6 +1004,7 @@ oncat
 one
 onfig
 only
+ookup
 open
 opt/
 opted
@@ -1016,6 +1022,7 @@ ormat
 orph
 otorola
 ottle
+ound
 ously
 out
 over
@@ -1075,6 +1082,7 @@ pose
 posit
 possib
 post
+poun
 power
 pre_
 pred
@@ -1219,7 +1227,7 @@ rotat
 rotocol
 rottl
 rough
-round
+roun
 roup
 row
 rroga
@@ -1328,7 +1336,7 @@ solve
 some
 sony
 sort
-sound
+soun
 source
 space
 spacing
@@ -1584,7 +1592,7 @@ yield
 you
 zeppelin
 zero
-zing
 zigbee
+zing
 zona
 zorro

{credsweeper-1.13.0 → credsweeper-1.13.2}/credsweeper/credentials/line_data.py

@@ -197,15 +197,14 @@ class LineData:
         If line seem to be a URL - split by & character.
         Variable should be right most value after & or ? ([-1]). And value should be left most before & ([0])
         """
-        if self.check_url_part():
+        # skip sanitize in case of URL credential rule - the regex is mature enough
+        if self.check_url_part() and not self.variable.endswith("://"):
             # all checks have passed - line before the value may be a URL
             self.variable = self.variable.rsplit('&')[-1].rsplit('?')[-1].rsplit(';')[-1]
             self.value = self.value.split('&', maxsplit=1)[0].split(';', maxsplit=1)[0].split('#', maxsplit=1)[0]
-            if not self.variable.endswith("://"):
-                # skip sanitize in case of URL credential rule
-                self.value = self.url_unicode_split.split(self.value)[0]
-                if self._3d_escaped_separator:
-                    self.value = self.url_percent_split.split(self.value)[0]
+            self.value = self.url_unicode_split.split(self.value)[0]
+            if self._3d_escaped_separator:
+                self.value = self.url_percent_split.split(self.value)[0]
 
     def clean_bash_parameters(self) -> None:
         """Split variable and value by bash special characters, if line assumed to be CLI command."""

{credsweeper-1.13.0 → credsweeper-1.13.2}/credsweeper/deep_scanner/abstract_scanner.py

@@ -51,6 +51,7 @@ class AbstractScanner(ABC):
     @abstractmethod
     def get_deep_scanners(data: bytes, descriptor: Descriptor, depth: int) -> Tuple[List[Any], List[Any]]:
         """Returns possibly scan methods for the data depends on content and fallback scanners"""
+        raise NotImplementedError(__name__)
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 

credsweeper-1.13.2/credsweeper/deep_scanner/csv_scanner.py

@@ -0,0 +1,71 @@
+import csv
+import io
+import logging
+from abc import ABC
+from typing import List, Optional, Dict, Any
+
+from credsweeper.common.constants import MAX_LINE_LENGTH
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.struct_content_provider import StructContentProvider
+
+logger = logging.getLogger(__name__)
+
+
+class CsvScanner(AbstractScanner, ABC):
+    """Implements CSV scanning"""
+
+    sniffer = csv.Sniffer()
+    # do not use space as separator to avoid hallucinations
+    delimiters = ",;\t|\x1F"
+
+    @classmethod
+    def get_structure(cls, text: str) -> List[Dict[str, Any]]:
+        """Reads a text as CSV standard with guessed dialect"""
+        # windows style \r\n
+        first_line_end = text.find('\r', 0, MAX_LINE_LENGTH)
+        line_terminator = "\r\n"
+        if 0 > first_line_end:
+            # unix style \n
+            first_line_end = text.find('\n', 0, MAX_LINE_LENGTH)
+            line_terminator = "\n"
+            if 0 > first_line_end:
+                raise ValueError(f"No suitable line end found in {MAX_LINE_LENGTH} symbols")
+
+        first_line = text[:first_line_end]
+        dialect = cls.sniffer.sniff(first_line, delimiters=cls.delimiters)
+        rows = []
+        reader = csv.DictReader(io.StringIO(text),
+                                delimiter=dialect.delimiter,
+                                lineterminator=line_terminator,
+                                strict=True)
+        # check the constant columns number for all rows
+        fields_number = sum(1 for x in reader.fieldnames if x is not None)
+        for row in reader:
+            if not isinstance(row, dict):
+                raise ValueError(f"ERROR: wrong row '{row}'")
+            if len(row) != fields_number or any(x is None for x in row.values()):
+                # None means no separator used
+                raise ValueError(f"Different columns number in row '{row}' - mismatch {fields_number}")
+            rows.append(row)
+        return rows
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Tries to scan each row as structure with column name in key"""
+        try:
+            if rows := self.get_structure(data_provider.text):
+                struct_content_provider = StructContentProvider(struct=rows,
+                                                                file_path=data_provider.file_path,
+                                                                file_type=data_provider.file_type,
+                                                                info=f"{data_provider.info}|CSV")
+                new_limit = recursive_limit_size - sum(len(x) for x in rows)
+                struct_candidates = self.structure_scan(struct_content_provider, depth, new_limit)
+                return struct_candidates
+        except Exception as csv_exc:
+            logger.debug(f"{data_provider.file_path}:{csv_exc}")
+        return None

{credsweeper-1.13.0 → credsweeper-1.13.2}/credsweeper/deep_scanner/deep_scanner.py

@@ -6,6 +6,7 @@ from credsweeper.scanner.scanner import Scanner
 from credsweeper.utils.util import Util
 from .byte_scanner import ByteScanner
 from .bzip2_scanner import Bzip2Scanner
+from .csv_scanner import CsvScanner
 from .deb_scanner import DebScanner
 from .docx_scanner import DocxScanner
 from .eml_scanner import EmlScanner
@@ -39,6 +40,7 @@ class DeepScanner(
     ByteScanner,  #
     Bzip2Scanner,  #
     DocxScanner,  #
+    CsvScanner,  #
     EncoderScanner,  #
     GzipScanner,  #
     HtmlScanner,  #
@@ -160,16 +162,18 @@ class DeepScanner(
                 deep_scanners.append(EmlScanner)
             else:
                 if 0 < depth:
-                    # formal patch looks like an eml
+                    # a formal patch looks like an eml
                     deep_scanners.append(PatchScanner)
                 fallback_scanners.append(EmlScanner)
             fallback_scanners.append(ByteScanner)
         elif not Util.is_binary(data):
+            # keep ByteScanner first to apply real value position if possible
+            deep_scanners.append(ByteScanner)
             if 0 < depth:
                 deep_scanners.append(PatchScanner)
                 deep_scanners.append(EncoderScanner)
                 deep_scanners.append(LangScanner)
-            deep_scanners.append(ByteScanner)
+                deep_scanners.append(CsvScanner)
         else:
             if 0 < depth:
                 deep_scanners.append(StringsScanner)

{credsweeper-1.13.0 → credsweeper-1.13.2}/credsweeper/deep_scanner/jks_scanner.py

@@ -4,6 +4,7 @@ from typing import List, Optional
 
 import jks
 
+from credsweeper.common.constants import Severity, Confidence
 from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
@@ -24,14 +25,22 @@ class JksScanner(AbstractScanner, ABC):
             try:
                 keystore = jks.KeyStore.loads(data_provider.data, pw_probe, try_decrypt_keys=True)
                 # the password probe has passed, it will be the value
-                info = (f"{data_provider.info}|JKS:"
-                        f"{'sensitive data' if keystore.private_keys or keystore.secret_keys else 'default password'}")
+                if keystore.private_keys or keystore.secret_keys:
+                    severity = Severity.HIGH
+                    confidence = Confidence.STRONG
+                    info = f"{data_provider.info}|JKS:default password"
+                else:
+                    severity = Severity.LOW
+                    confidence = Confidence.WEAK
+                    info = f"{data_provider.info}|JKS:sensitive data"
                 candidate = Candidate.get_dummy_candidate(
                     self.config,  #
                     data_provider.file_path,  #
                     data_provider.file_type,  #
                     info,  #
                     "Java Key Storage")
+                candidate.severity = severity
+                candidate.confidence = confidence
                 value = pw_probe or "<EMPTY PASSWORD>"
                 candidate.line_data_list[0].line = f"'{value}' is the password"
                 candidate.line_data_list[0].value = pw_probe or "<EMPTY PASSWORD>"

{credsweeper-1.13.0 → credsweeper-1.13.2}/credsweeper/deep_scanner/pkcs_scanner.py

@@ -3,6 +3,7 @@ import logging
 from abc import ABC
 from typing import List, Optional
 
+from credsweeper.common.constants import Severity, Confidence
 from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
@@ -35,6 +36,9 @@ class PkcsScanner(AbstractScanner, ABC):
                         "PKCS")
                     candidate.line_data_list[0].line = base64.b64encode(data_provider.data).decode()
                     candidate.line_data_list[0].value = repr(password)
+                    # high severity is assigned to private key rules
+                    candidate.severity = Severity.HIGH
+                    candidate.confidence = Confidence.STRONG
                     return [candidate]
             except Exception as pkcs_exc:
                 logger.debug(f"{data_provider.file_path}:{pw_probe}:{pkcs_exc}")