credsweeper 1.11.5__py3-none-any.whl → 1.13.3__py3-none-any.whl

credsweeper/common/morpheme_checklist.txt

@@ -14,11 +14,15 @@
 /var
 000
 111
+14159265
+18284590
 222
 333
 444
 555
+65358979
 666
+71828182
 777
 80211
 888
@@ -195,7 +199,7 @@ aux
 avail
 avatar
 aver
-awesome
+awesom
 axis
 azure
 back
@@ -227,12 +231,14 @@ bind
 bio
 bipol
 bit
+bixby
 black
 blan
 bless
 blic
 blish
 blob
+blood
 blue
 board
 bob
@@ -243,7 +249,7 @@ boost
 boot
 boss
 bot
-bound
+boun
 box
 branch
 break
@@ -497,6 +503,7 @@ dust
 dvb
 dynamic
 dynamo
+eadbee
 easin
 easy
 ecdhe
@@ -607,6 +614,7 @@ fleet
 flick
 flix
 float
+flood
 floor
 fluent
 fluid
@@ -615,7 +623,7 @@ focus
 foo
 for
 fossil
-found
+foun
 fpga
 frame
 free
@@ -648,6 +656,7 @@ git
 given
 global
 gobble
+good
 google
 grab
 grace
@@ -703,6 +712,7 @@ home
 hook
 horizon
 host
+houn
 hours
 html
 http
@@ -789,6 +799,7 @@ jpg_
 json
 jump
 justif
+kafka
 kerberos
 kernel
 key
@@ -797,6 +808,8 @@ kill
 kind
 kinesis
 kirk
+know
+knox
 kris
 lab
 lag
@@ -853,7 +866,7 @@ local
 lock
 log
 long
-lookup
+look
 loop
 loose
 lost
@@ -946,6 +959,7 @@ ndow
 ned
 need
 neigh
+neo4j
 ner
 net
 neutr
@@ -990,6 +1004,7 @@ oncat
 one
 onfig
 only
+ookup
 open
 opt/
 opted
@@ -1007,6 +1022,7 @@ ormat
 orph
 otorola
 ottle
+ound
 ously
 out
 over
@@ -1066,6 +1082,7 @@ pose
 posit
 possib
 post
+poun
 power
 pre_
 pred
@@ -1210,7 +1227,7 @@ rotat
 rotocol
 rottl
 rough
-round
+roun
 roup
 row
 rroga
@@ -1222,6 +1239,7 @@ run
 rxtx
 sabl
 sage
+salt
 same
 sampl
 sams
@@ -1315,9 +1333,10 @@ sock
 soft
 solid
 solve
+some
 sony
 sort
-sound
+soun
 source
 space
 spacing
@@ -1427,6 +1446,7 @@ tio
 tish
 title
 titud
+tizen
 tmp/
 to_
 tod
@@ -1438,6 +1458,7 @@ topic
 tory
 total
 touch
+tour
 trace
 tract
 traffic
@@ -1571,6 +1592,7 @@ yield
 you
 zeppelin
 zero
+zigbee
 zing
 zona
 zorro

credsweeper/config/__init__.py

@@ -1 +0,0 @@
-from credsweeper.config.config import Config

credsweeper/config/config.py

@@ -4,7 +4,7 @@ from typing import Dict, List, Optional, Set, Any
 from humanfriendly import parse_size
 
 from credsweeper.common.constants import Severity, DEFAULT_PATTERN_LEN
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 
 class Config:
@@ -35,12 +35,13 @@ class Config:
         self.candidate_output: List[str] = config["candidate_output"]
         self.find_by_ext: bool = config["find_by_ext"]
         self.size_limit: Optional[int] = parse_size(config["size_limit"]) if config["size_limit"] is not None else None
+        self.pedantic: bool = bool(config["pedantic"])
         self.depth: int = int(config["depth"])
         self.doc: bool = config["doc"]
         self.severity: Severity = Severity.get(config.get("severity"))
 
-        self.min_keyword_value_length: int = int(config["min_keyword_value_length"])
-        self.min_pattern_value_length: int = int(config["min_pattern_value_length"])
+        self.max_url_cred_value_length: int = int(config["max_url_cred_value_length"])
+        self.max_password_value_length: int = int(config["max_password_value_length"])
 
         # Trim exclude patterns from space like characters
         self.exclude_lines = set(line.strip() for line in self.exclude_lines)

credsweeper/credentials/__init__.py

@@ -1,5 +0,0 @@
-from credsweeper.credentials.candidate import Candidate
-from credsweeper.credentials.candidate_group_generator import CandidateGroupGenerator
-from credsweeper.credentials.candidate_key import CandidateKey
-from credsweeper.credentials.credential_manager import CredentialManager
-from credsweeper.credentials.line_data import LineData

credsweeper/credentials/augment_candidates.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 
 
 def augment_candidates(candidates: List[Candidate], new_candidates: List[Candidate]):

credsweeper/credentials/candidate.py

@@ -4,7 +4,7 @@ from json.encoder import py_encode_basestring_ascii
 from typing import Any, Dict, List, Optional
 
 from credsweeper.common.constants import Severity, Confidence
-from credsweeper.config import Config
+from credsweeper.config.config import Config
 from credsweeper.credentials.line_data import LineData
 
 

credsweeper/credentials/credential_manager.py

@@ -2,7 +2,7 @@ import logging
 from multiprocessing import Manager
 from typing import List, Dict, Tuple
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.credentials.candidate_group_generator import CandidateGroupGenerator, CandidateKey
 
 logger = logging.getLogger(__name__)

credsweeper/credentials/line_data.py

@@ -8,8 +8,8 @@ from typing import Any, Dict, Optional, Tuple
 from colorama import Fore, Style
 
 from credsweeper.common.constants import MAX_LINE_LENGTH, UTF_8, StartEnd, ML_HUNK
-from credsweeper.config import Config
-from credsweeper.utils import Util
+from credsweeper.config.config import Config
+from credsweeper.utils.util import Util
 
 
 class LineData:
@@ -137,12 +137,33 @@ class LineData:
 
     def sanitize_value(self):
         """Clean found value from extra artifacts. Correct positions if changed."""
+        # process the quotation workaround before cached properties invocation
+        if not self.value_leftquote and not self.value_rightquote:
+            while self.value:
+                first_symbol_code = ord(self.value[0])
+                last_symbol_code = ord(self.value[-1])
+                if 0x2018 <= first_symbol_code <= 0x201B and 0x2018 <= last_symbol_code <= 0x201B:
+                    self.value_leftquote = self.value_rightquote = "'"
+                    self.value = self.value[:-1]
+                    self.value_end -= 1
+                    self.value = self.value[1:]
+                    self.value_start += 1
+                elif 0x201C <= first_symbol_code <= 0x201F and 0x201C <= last_symbol_code <= 0x201F:
+                    self.value_leftquote = self.value_rightquote = '"'
+                    self.value = self.value[1:]
+                    self.value_start += 1
+                    self.value = self.value[:-1]
+                    self.value_end -= 1
+                else:
+                    break
+
         if self.variable and self.value and not self.is_well_quoted_value:
             # sanitize is actual step for keyword pattern only
             _value = self.value
             self.clean_url_parameters()
             self.clean_bash_parameters()
             self.clean_toml_parameters()
+            self.clean_tag_parameters()
             if 0 <= self.value_start and 0 <= self.value_end and len(self.value) < len(_value):
                 start = _value.find(self.value)
                 self.value_start += start
@@ -176,15 +197,14 @@ class LineData:
         If line seem to be a URL - split by & character.
         Variable should be right most value after & or ? ([-1]). And value should be left most before & ([0])
         """
-        if self.check_url_part():
+        # skip sanitize in case of URL credential rule - the regex is mature enough
+        if self.check_url_part() and not self.variable.endswith("://"):
             # all checks have passed - line before the value may be a URL
             self.variable = self.variable.rsplit('&')[-1].rsplit('?')[-1].rsplit(';')[-1]
             self.value = self.value.split('&', maxsplit=1)[0].split(';', maxsplit=1)[0].split('#', maxsplit=1)[0]
-            if not self.variable.endswith("://"):
-                # skip sanitize in case of URL credential rule
-                self.value = self.url_unicode_split.split(self.value)[0]
-                if self._3d_escaped_separator:
-                    self.value = self.url_percent_split.split(self.value)[0]
+            self.value = self.url_unicode_split.split(self.value)[0]
+            if self._3d_escaped_separator:
+                self.value = self.url_percent_split.split(self.value)[0]
 
     def clean_bash_parameters(self) -> None:
         """Split variable and value by bash special characters, if line assumed to be CLI command."""
@@ -212,6 +232,21 @@ class LineData:
                     self.value = self.value[:-1]
                     cleaning_required = True
 
+    def clean_tag_parameters(self) -> None:
+        """Remove closing tag from value if the opened is somewhere before in line"""
+        cleaning_required = self.value and self.value.endswith('>')
+        while cleaning_required:
+            closing_tag_pos = self.value.rfind("</")
+            if 0 <= closing_tag_pos:
+                # use `<a` to avoid tag parameters
+                opening_tag_prefix = f"<{self.value[closing_tag_pos + 2:-1]}"
+                if cleaning_required := (opening_tag_prefix not in self.value
+                                         and 0 <= self.line.find(opening_tag_prefix, 0, self.value_start)):
+                    self.value = self.value[:closing_tag_pos]
+                    cleaning_required = self.value and self.value.endswith('>')
+            else:
+                break
+
     def sanitize_variable(self) -> None:
         """Remove trailing spaces, dashes and quotations around the variable. Correct position."""
         sanitized_var_len = 0

credsweeper/deep_scanner/__init__.py

@@ -1 +0,0 @@
-from credsweeper.deep_scanner.deep_scanner import DeepScanner

credsweeper/deep_scanner/abstract_scanner.py

@@ -6,9 +6,9 @@ from typing import List, Optional, Tuple, Any, Generator
 
 from credsweeper.common.constants import RECURSIVE_SCAN_LIMITATION, MIN_DATA_LEN, DEFAULT_ENCODING, UTF_8, \
     MIN_VALUE_LENGTH
-from credsweeper.config import Config
-from credsweeper.credentials import Candidate
+from credsweeper.config.config import Config
 from credsweeper.credentials.augment_candidates import augment_candidates
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.file_handler.byte_content_provider import ByteContentProvider
 from credsweeper.file_handler.content_provider import ContentProvider
 from credsweeper.file_handler.data_content_provider import DataContentProvider
@@ -18,7 +18,7 @@ from credsweeper.file_handler.file_path_extractor import FilePathExtractor
 from credsweeper.file_handler.string_content_provider import StringContentProvider
 from credsweeper.file_handler.struct_content_provider import StructContentProvider
 from credsweeper.file_handler.text_content_provider import TextContentProvider
-from credsweeper.scanner import Scanner
+from credsweeper.scanner.scanner import Scanner
 
 logger = logging.getLogger(__name__)
 
@@ -51,6 +51,7 @@ class AbstractScanner(ABC):
     @abstractmethod
     def get_deep_scanners(data: bytes, descriptor: Descriptor, depth: int) -> Tuple[List[Any], List[Any]]:
         """Returns possibly scan methods for the data depends on content and fallback scanners"""
+        raise NotImplementedError(__name__)
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 

credsweeper/deep_scanner/byte_scanner.py

@@ -2,7 +2,7 @@ import logging
 from abc import ABC
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.file_handler.byte_content_provider import ByteContentProvider
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from .abstract_scanner import AbstractScanner

credsweeper/deep_scanner/bzip2_scanner.py

@@ -4,10 +4,10 @@ from abc import ABC
 from pathlib import Path
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/deep_scanner/csv_scanner.py

@@ -0,0 +1,71 @@
+import csv
+import io
+import logging
+from abc import ABC
+from typing import List, Optional, Dict, Any
+
+from credsweeper.common.constants import MAX_LINE_LENGTH
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.struct_content_provider import StructContentProvider
+
+logger = logging.getLogger(__name__)
+
+
+class CsvScanner(AbstractScanner, ABC):
+    """Implements CSV scanning"""
+
+    sniffer = csv.Sniffer()
+    # do not use space as separator to avoid hallucinations
+    delimiters = ",;\t|\x1F"
+
+    @classmethod
+    def get_structure(cls, text: str) -> List[Dict[str, Any]]:
+        """Reads a text as CSV standard with guessed dialect"""
+        # windows style \r\n
+        first_line_end = text.find('\r', 0, MAX_LINE_LENGTH)
+        line_terminator = "\r\n"
+        if 0 > first_line_end:
+            # unix style \n
+            first_line_end = text.find('\n', 0, MAX_LINE_LENGTH)
+            line_terminator = "\n"
+            if 0 > first_line_end:
+                raise ValueError(f"No suitable line end found in {MAX_LINE_LENGTH} symbols")
+
+        first_line = text[:first_line_end]
+        dialect = cls.sniffer.sniff(first_line, delimiters=cls.delimiters)
+        rows = []
+        reader = csv.DictReader(io.StringIO(text),
+                                delimiter=dialect.delimiter,
+                                lineterminator=line_terminator,
+                                strict=True)
+        # check the constant columns number for all rows
+        fields_number = sum(1 for x in reader.fieldnames if x is not None)
+        for row in reader:
+            if not isinstance(row, dict):
+                raise ValueError(f"ERROR: wrong row '{row}'")
+            if len(row) != fields_number or any(x is None for x in row.values()):
+                # None means no separator used
+                raise ValueError(f"Different columns number in row '{row}' - mismatch {fields_number}")
+            rows.append(row)
+        return rows
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Tries to scan each row as structure with column name in key"""
+        try:
+            if rows := self.get_structure(data_provider.text):
+                struct_content_provider = StructContentProvider(struct=rows,
+                                                                file_path=data_provider.file_path,
+                                                                file_type=data_provider.file_type,
+                                                                info=f"{data_provider.info}|CSV")
+                new_limit = recursive_limit_size - sum(len(x) for x in rows)
+                struct_candidates = self.structure_scan(struct_content_provider, depth, new_limit)
+                return struct_candidates
+        except Exception as csv_exc:
+            logger.debug(f"{data_provider.file_path}:{csv_exc}")
+        return None

credsweeper/deep_scanner/deb_scanner.py

@@ -4,7 +4,7 @@ from abc import ABC
 from typing import List, Optional, Generator, Tuple
 
 from credsweeper.common.constants import MIN_DATA_LEN, UTF_8
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.utils.util import Util

credsweeper/deep_scanner/deep_scanner.py

@@ -1,12 +1,12 @@
 import logging
 from typing import List, Any, Tuple
 
-from credsweeper.common.constants import MIN_DATA_LEN
-from credsweeper.config import Config
-from credsweeper.scanner import Scanner
-from credsweeper.utils import Util
+from credsweeper.config.config import Config
+from credsweeper.scanner.scanner import Scanner
+from credsweeper.utils.util import Util
 from .byte_scanner import ByteScanner
 from .bzip2_scanner import Bzip2Scanner
+from .csv_scanner import CsvScanner
 from .deb_scanner import DebScanner
 from .docx_scanner import DocxScanner
 from .eml_scanner import EmlScanner
@@ -23,7 +23,9 @@ from .pdf_scanner import PdfScanner
 from .pkcs_scanner import PkcsScanner
 from .pptx_scanner import PptxScanner
 from .rpm_scanner import RpmScanner
+from .rtf_scanner import RtfScanner
 from .sqlite3_scanner import Sqlite3Scanner
+from .strings_scanner import StringsScanner
 from .tar_scanner import TarScanner
 from .tmx_scanner import TmxScanner
 from .xlsx_scanner import XlsxScanner
@@ -38,6 +40,7 @@ class DeepScanner(
     ByteScanner,  #
     Bzip2Scanner,  #
     DocxScanner,  #
+    CsvScanner,  #
     EncoderScanner,  #
     GzipScanner,  #
     HtmlScanner,  #
@@ -49,8 +52,10 @@ class DeepScanner(
     PdfScanner,  #
     PkcsScanner,  #
     PptxScanner,  #
+    RtfScanner,  #
     RpmScanner,  #
     Sqlite3Scanner,  #
+    StringsScanner,  #
     TarScanner,  #
     DebScanner,  #
     XmlScanner,  #
@@ -133,6 +138,9 @@ class DeepScanner(
                 deep_scanners.append(Sqlite3Scanner)
         elif Util.is_asn1(data):
             deep_scanners.append(PkcsScanner)
+        elif Util.is_rtf(data):
+            deep_scanners.append(RtfScanner)
+            fallback_scanners.append(ByteScanner)
         elif Util.is_xml(data):
             if Util.is_html(data):
                 deep_scanners.append(HtmlScanner)
@@ -150,24 +158,26 @@ class DeepScanner(
                 deep_scanners.append(XmlScanner)
                 fallback_scanners.append(ByteScanner)
         elif Util.is_eml(data):
-            if ".eml" == descriptor.extension:
+            if descriptor.extension in (".eml", ".mht"):
                 deep_scanners.append(EmlScanner)
             else:
                 if 0 < depth:
-                    # formal patch looks like an eml
+                    # a formal patch looks like an eml
                     deep_scanners.append(PatchScanner)
                 fallback_scanners.append(EmlScanner)
             fallback_scanners.append(ByteScanner)
-        elif Util.is_known(data):
-            # the format is known but cannot be scanned
-            pass
         elif not Util.is_binary(data):
+            # keep ByteScanner first to apply real value position if possible
+            deep_scanners.append(ByteScanner)
             if 0 < depth:
                 deep_scanners.append(PatchScanner)
                 deep_scanners.append(EncoderScanner)
                 deep_scanners.append(LangScanner)
-            deep_scanners.append(ByteScanner)
+                deep_scanners.append(CsvScanner)
         else:
-            logger.warning("Cannot apply a deep scanner for type %s prefix %s %d", descriptor,
-                           repr(data[:MIN_DATA_LEN]), len(data))
+            if 0 < depth:
+                deep_scanners.append(StringsScanner)
+            else:
+                logger.warning("Cannot apply a deep scanner for type %s prefix %s %d", descriptor, repr(data[:32]),
+                               len(data))
         return deep_scanners, fallback_scanners

credsweeper/deep_scanner/docx_scanner.py

@@ -11,7 +11,7 @@ from docx.table import _Cell, Table
 from docx.text.paragraph import Paragraph
 from lxml.etree import _Element
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.string_content_provider import StringContentProvider

credsweeper/deep_scanner/eml_scanner.py

@@ -3,7 +3,7 @@ import logging
 from abc import ABC
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.byte_content_provider import ByteContentProvider
 from credsweeper.file_handler.data_content_provider import DataContentProvider

credsweeper/deep_scanner/encoder_scanner.py

@@ -2,7 +2,7 @@ import logging
 from abc import ABC
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 

credsweeper/deep_scanner/gzip_scanner.py

@@ -5,10 +5,10 @@ from abc import ABC
 from pathlib import Path
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
-from credsweeper.utils import Util
+from credsweeper.utils.util import Util
 
 logger = logging.getLogger(__name__)
 

credsweeper/deep_scanner/html_scanner.py

@@ -2,7 +2,7 @@ import logging
 from abc import ABC
 from typing import List, Optional
 
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.string_content_provider import StringContentProvider

credsweeper/deep_scanner/jclass_scanner.py

@@ -5,7 +5,7 @@ from abc import ABC
 from typing import List, Optional
 
 from credsweeper.common.constants import MIN_DATA_LEN, UTF_8
-from credsweeper.credentials import Candidate
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 from credsweeper.file_handler.struct_content_provider import StructContentProvider

credsweeper/deep_scanner/jks_scanner.py

@@ -4,7 +4,8 @@ from typing import List, Optional
 
 import jks
 
-from credsweeper.credentials import Candidate
+from credsweeper.common.constants import Severity, Confidence
+from credsweeper.credentials.candidate import Candidate
 from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
 from credsweeper.file_handler.data_content_provider import DataContentProvider
 
@@ -24,14 +25,22 @@ class JksScanner(AbstractScanner, ABC):
             try:
                 keystore = jks.KeyStore.loads(data_provider.data, pw_probe, try_decrypt_keys=True)
                 # the password probe has passed, it will be the value
-                info = (f"{data_provider.info}|JKS:"
-                        f"{'sensitive data' if keystore.private_keys or keystore.secret_keys else 'default password'}")
+                if keystore.private_keys or keystore.secret_keys:
+                    severity = Severity.HIGH
+                    confidence = Confidence.STRONG
+                    info = f"{data_provider.info}|JKS:default password"
+                else:
+                    severity = Severity.LOW
+                    confidence = Confidence.WEAK
+                    info = f"{data_provider.info}|JKS:sensitive data"
                 candidate = Candidate.get_dummy_candidate(
                     self.config,  #
                     data_provider.file_path,  #
                     data_provider.file_type,  #
                     info,  #
                     "Java Key Storage")
+                candidate.severity = severity
+                candidate.confidence = confidence
                 value = pw_probe or "<EMPTY PASSWORD>"
                 candidate.line_data_list[0].line = f"'{value}' is the password"
                 candidate.line_data_list[0].value = pw_probe or "<EMPTY PASSWORD>"