credsweeper 1.13.1__tar.gz → 1.13.2__tar.gz

{credsweeper-1.13.1 → credsweeper-1.13.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: credsweeper
-Version: 1.13.1
+Version: 1.13.2
 Summary: Credential Sweeper
 Project-URL: Homepage, https://github.com/Samsung/CredSweeper
 Project-URL: Bug Tracker, https://github.com/Samsung/CredSweeper/issues

{credsweeper-1.13.1 → credsweeper-1.13.2}/credsweeper/__init__.py

@@ -24,4 +24,4 @@ __all__ = [
     "__version__"
 ]
 
-__version__ = "1.13.1"
+__version__ = "1.13.2"

{credsweeper-1.13.1 → credsweeper-1.13.2}/credsweeper/deep_scanner/abstract_scanner.py

@@ -51,6 +51,7 @@ class AbstractScanner(ABC):
     @abstractmethod
     def get_deep_scanners(data: bytes, descriptor: Descriptor, depth: int) -> Tuple[List[Any], List[Any]]:
         """Returns possibly scan methods for the data depends on content and fallback scanners"""
+        raise NotImplementedError(__name__)
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 

credsweeper-1.13.2/credsweeper/deep_scanner/csv_scanner.py

@@ -0,0 +1,71 @@
+import csv
+import io
+import logging
+from abc import ABC
+from typing import List, Optional, Dict, Any
+
+from credsweeper.common.constants import MAX_LINE_LENGTH
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.struct_content_provider import StructContentProvider
+
+logger = logging.getLogger(__name__)
+
+
+class CsvScanner(AbstractScanner, ABC):
+    """Implements CSV scanning"""
+
+    sniffer = csv.Sniffer()
+    # do not use space as separator to avoid hallucinations
+    delimiters = ",;\t|\x1F"
+
+    @classmethod
+    def get_structure(cls, text: str) -> List[Dict[str, Any]]:
+        """Reads a text as CSV standard with guessed dialect"""
+        # windows style \r\n
+        first_line_end = text.find('\r', 0, MAX_LINE_LENGTH)
+        line_terminator = "\r\n"
+        if 0 > first_line_end:
+            # unix style \n
+            first_line_end = text.find('\n', 0, MAX_LINE_LENGTH)
+            line_terminator = "\n"
+            if 0 > first_line_end:
+                raise ValueError(f"No suitable line end found in {MAX_LINE_LENGTH} symbols")
+
+        first_line = text[:first_line_end]
+        dialect = cls.sniffer.sniff(first_line, delimiters=cls.delimiters)
+        rows = []
+        reader = csv.DictReader(io.StringIO(text),
+                                delimiter=dialect.delimiter,
+                                lineterminator=line_terminator,
+                                strict=True)
+        # check the constant columns number for all rows
+        fields_number = sum(1 for x in reader.fieldnames if x is not None)
+        for row in reader:
+            if not isinstance(row, dict):
+                raise ValueError(f"ERROR: wrong row '{row}'")
+            if len(row) != fields_number or any(x is None for x in row.values()):
+                # None means no separator used
+                raise ValueError(f"Different columns number in row '{row}' - mismatch {fields_number}")
+            rows.append(row)
+        return rows
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Tries to scan each row as structure with column name in key"""
+        try:
+            if rows := self.get_structure(data_provider.text):
+                struct_content_provider = StructContentProvider(struct=rows,
+                                                                file_path=data_provider.file_path,
+                                                                file_type=data_provider.file_type,
+                                                                info=f"{data_provider.info}|CSV")
+                new_limit = recursive_limit_size - sum(len(x) for x in rows)
+                struct_candidates = self.structure_scan(struct_content_provider, depth, new_limit)
+                return struct_candidates
+        except Exception as csv_exc:
+            logger.debug(f"{data_provider.file_path}:{csv_exc}")
+        return None

{credsweeper-1.13.1 → credsweeper-1.13.2}/credsweeper/deep_scanner/deep_scanner.py

@@ -6,6 +6,7 @@ from credsweeper.scanner.scanner import Scanner
 from credsweeper.utils.util import Util
 from .byte_scanner import ByteScanner
 from .bzip2_scanner import Bzip2Scanner
+from .csv_scanner import CsvScanner
 from .deb_scanner import DebScanner
 from .docx_scanner import DocxScanner
 from .eml_scanner import EmlScanner
@@ -39,6 +40,7 @@ class DeepScanner(
     ByteScanner,  #
     Bzip2Scanner,  #
     DocxScanner,  #
+    CsvScanner,  #
     EncoderScanner,  #
     GzipScanner,  #
     HtmlScanner,  #
@@ -160,16 +162,18 @@ class DeepScanner(
                 deep_scanners.append(EmlScanner)
             else:
                 if 0 < depth:
-                    # formal patch looks like an eml
+                    # a formal patch looks like an eml
                     deep_scanners.append(PatchScanner)
                 fallback_scanners.append(EmlScanner)
             fallback_scanners.append(ByteScanner)
         elif not Util.is_binary(data):
+            # keep ByteScanner first to apply real value position if possible
+            deep_scanners.append(ByteScanner)
             if 0 < depth:
                 deep_scanners.append(PatchScanner)
                 deep_scanners.append(EncoderScanner)
                 deep_scanners.append(LangScanner)
-            deep_scanners.append(ByteScanner)
+                deep_scanners.append(CsvScanner)
         else:
             if 0 < depth:
                 deep_scanners.append(StringsScanner)