txt2stix 0.0.4__py3-none-any.whl

txt2stix/pattern/extractors/crypto/btc_extractor.py

@@ -0,0 +1,38 @@
+from validators import base58
+from ..base_extractor import BaseExtractor
+from base58 import b58decode
+
+
+class CryptoBTCWalletExtractor(BaseExtractor):
+    """
+    A class for extracting Bitcoin (BTC) wallet addresses from text using regular expressions.
+
+    """
+
+    name = "pattern_cryptocurrency_btc_wallet"
+    extraction_regex = "^(bc1|[13])[a-km-zA-HJ-NP-Z1-9]{25,34}$"
+
+    @classmethod
+    def filter_function(cls, value):
+        try:
+            b58decode(value).hex()
+            return True
+        except:
+            return False
+
+class CryptoBTCWalletTransactionExtractor(CryptoBTCWalletExtractor):
+    """
+    A class for extracting Bitcoin (BTC) wallet addresses along with the transactions on that account.
+    
+    """
+    name = "pattern_cryptocurrency_btc_wallet_transaction"
+
+class CryptoBTCTransactionExtractor(BaseExtractor):
+    """
+    A class for extracting Bitcoin (BTC) transaction hash from text using regular expressions.
+
+    """
+
+    name = "pattern_cryptocurrency_btc_transaction"
+    extraction_regex = "^[a-fA-F0-9]{64}$"
+

txt2stix/pattern/extractors/directory/__init__.py

@@ -0,0 +1,10 @@
+from .unix_directory_extractor import UnixDirectoryExtractor
+from .unix_file_path_extractor import UnixFilePathExtractor
+from .windows_directory_path_extractor import WindowDirectoryExtractor
+from .windows_file_path_extractor import WindowsFilePathExtractor
+
+DIRECTORY_EXTRACTORS = [UnixDirectoryExtractor,
+                        UnixFilePathExtractor,
+                        WindowDirectoryExtractor,
+                        WindowsFilePathExtractor
+                        ]

txt2stix/pattern/extractors/directory/unix_directory_extractor.py

@@ -0,0 +1,40 @@
+from ..base_extractor import BaseExtractor
+from pathvalidate import is_valid_filepath
+
+
+class UnixDirectoryExtractor(BaseExtractor):
+    """
+    A class for extracting valid Unix-style directory paths from text using a custom extraction function.
+
+    Attributes:
+        name (str): The name of the extractor, set to "unix-directory-path".
+        extraction_function (function): The custom extraction function to validate and extract directory paths.
+    """
+
+    name = "pattern_directory_unix"
+    extraction_function = lambda x: UnixDirectoryExtractor.is_valid_directory(x)
+
+    @staticmethod
+    def is_valid_directory(directory_path):
+        """
+        Custom extraction function to validate if the provided path is a valid Unix directory path.
+
+        Args:
+            directory_path (str): The path to be checked.
+
+        Returns:
+            bool: True if the path is a valid Unix directory path, False otherwise.
+        """
+        if "/" in directory_path and (directory_path[0] in ['.', '/', '~']):
+            try:
+                # Checking if it's a file path by splitting the path and checking the last component for a dot.
+                file_name = directory_path.split('/')[-1]
+                if "." in file_name:
+                    return False
+            except Exception as e:
+                pass
+
+            # Using pathvalidate library to check if the path is valid for Linux platform.
+            check = is_valid_filepath(directory_path, platform="Linux")
+            return check
+        return False

txt2stix/pattern/extractors/directory/unix_file_path_extractor.py

@@ -0,0 +1,42 @@
+from pathvalidate import is_valid_filepath
+
+from ..base_extractor import BaseExtractor
+from ..helper import validate_file_extension
+
+
+class UnixFilePathExtractor(BaseExtractor):
+    """
+    A class for extracting valid Unix-style file paths from text using a custom extraction function.
+
+    Attributes:
+        name (str): The name of the extractor, set to "unix-file-directory".
+        ignore_list (list): A list of strings to be ignored during extraction.
+        extraction_function (function): The custom extraction function to validate and extract file paths.
+    """
+
+    name = "pattern_directory_unix_file"
+    ignore_list = ["http://", "https://", "http[:]//", "http[://"]
+    extraction_function = lambda x: UnixFilePathExtractor.is_valid_directory(x)
+
+    @staticmethod
+    def is_valid_directory(directory_path):
+        """
+        Custom extraction function to validate if the provided path is a valid Unix file path.
+
+        Args:
+            directory_path (str): The path to be checked.
+
+        Returns:
+            bool: True if the path is a valid Unix file path, False otherwise.
+        """
+        if "/" in directory_path and (directory_path[0] in ['.', '/', '~']) and "\\" not in directory_path:
+            # Using pathvalidate library to check if the path is valid for Linux platform.
+            check = is_valid_filepath(directory_path, platform="Linux")
+            if check:
+                try:
+                    # Checking if it's a file path by splitting the path and checking the last component for a dot.
+                    if validate_file_extension(directory_path):
+                        return True
+                except Exception as e:
+                    return False
+        return False

txt2stix/pattern/extractors/directory/windows_directory_path_extractor.py

@@ -0,0 +1,47 @@
+from pathvalidate import is_valid_filepath
+from ..base_extractor import BaseExtractor
+from ..helper import validate_file_extension
+
+
+class WindowDirectoryExtractor(BaseExtractor):
+    """
+    A class for extracting valid Windows-style directory paths from text using a custom extraction function.
+
+    Attributes:
+        name (str): The name of the extractor, set to "widows-directory".
+        ignore_list (list): A list of strings to be ignored during extraction.
+        extraction_function (function): The custom extraction function to validate and extract directory paths.
+    """
+
+    name = "pattern_directory_windows"
+    extraction_function = lambda x: WindowDirectoryExtractor.is_valid_directory(x)
+
+    @staticmethod
+    def is_valid_directory(directory_path):
+        """
+        Custom extraction function to validate if the provided path is a valid Windows directory path.
+
+        Args:
+            directory_path (str): The path to be checked.
+
+        Returns:
+            bool: True if the path is a valid Windows directory path, False otherwise.
+        """
+        if directory_path == '\\' or directory_path == '\\\\':
+            return False
+
+        directory_path = directory_path.strip('"')
+        drive_letters = ["{}:\\".format(letter) for letter in "CDEFGHIJKLMNO"] + ["{}:".format(letter) for letter in "CDEFGHIJKLMNO"] + ['..\\', '\\', '\\\\']
+        flag = False
+        for prefix in drive_letters:
+            if directory_path.startswith(str(prefix)):
+                flag = True
+                break
+        start, _, _ = directory_path.partition('\\')
+        if start.startswith('%') and start.endswith('%'):
+            flag = True
+        if flag:
+            check = is_valid_filepath(directory_path, platform="Windows")
+            if not validate_file_extension(directory_path) and check:
+                return check
+        return False

txt2stix/pattern/extractors/directory/windows_file_path_extractor.py

@@ -0,0 +1,42 @@
+from pathvalidate import is_valid_filepath
+
+from txt2stix import utils
+from ..base_extractor import BaseExtractor
+from ..helper import validate_file_extension
+
+
+class WindowsFilePathExtractor(BaseExtractor):
+    """
+    WindowsFilePathExtractor is a class that extracts file paths on the Windows platform and validates the file extension.
+
+    Attributes:
+        name (str): The name of the extractor.
+        extraction_function (function): The function to extract file paths.
+    """
+    name = "pattern_directory_windows_with_file"
+    extraction_function = lambda x: WindowsFilePathExtractor.is_valid_directory(x)
+
+    @staticmethod
+    def is_valid_directory(directory_path):
+        """
+        Custom extraction function to validate if the provided path is a valid Windows directory path.
+
+        Args:
+            directory_path (str): The path to be checked.
+
+        Returns:
+            bool: True if the path is a valid Windows directory path, False otherwise.
+        """
+        directory_path = directory_path.strip('"')
+        drive_letters = ["{}:\\".format(letter) for letter in "CDEFGHIJKLMNO"] + ["{}:".format(letter) for letter in "CDEFGHIJKLMNO"] + ['..\\', '\\', '\\\\' ]
+        flag = False
+        for prefix in drive_letters:
+            if directory_path.startswith(str(prefix)):
+                flag = True
+                break
+        if flag:
+            check = is_valid_filepath(directory_path, platform="Windows")
+            if utils.validate_file_mimetype(directory_path) and check:
+                return True
+        return False
+

txt2stix/pattern/extractors/domain/__init__.py

@@ -0,0 +1,8 @@
+from .domain_extractor import DomainNameExtractor
+from .sub_domain_extractor import SubDomainExtractor
+from .hostname_extractor import HostnameBaseExtractor
+
+DOMAIN_EXTRACTORS = [DomainNameExtractor,
+                     SubDomainExtractor,
+                     HostnameBaseExtractor
+                     ]

txt2stix/pattern/extractors/domain/domain_extractor.py

@@ -0,0 +1,39 @@
+from tld import get_tld
+
+from ..base_extractor import BaseExtractor
+
+
+class DomainNameExtractor(BaseExtractor):
+    """
+    A class for extracting valid domain names from text using a custom extraction function.
+
+    Attributes:
+        name (str): The name of the extractor, set to "domain".
+        extraction_regex (function): The custom extraction function to validate and extract domain names.
+    """
+    name = "pattern_domain_name_only"
+    extraction_regex =  r'(([\da-zA-Z])([_\w-]{,62})\.){,127}(([\da-zA-Z])[_\w-]{,61})?([\da-zA-Z]\.((xn\-\-[a-zA-Z\d]+)|([a-zA-Z\d]{2,})))'
+
+    @staticmethod
+    def filter_function(domain):
+        """
+        Checks if the given domain is valid based on the number of dots and the top-level domain.
+
+        Args:
+            domain (str): The domain to be checked.
+
+        Returns:
+            bool: True if the domain is valid (has at most 2 dots and a valid TLD), False otherwise.
+        """
+        if domain.count('.') <= 2:
+            tld = get_tld(domain, fix_protocol=True, fail_silently=True)
+            if tld:
+                domain_name = domain.strip(f".{tld}")
+                if domain_name.count(".") == 0:
+                    return True
+        return False
+
+
+# class HostNameExtractor(DomainNameExtractor):
+#     filter_function = None
+#     name = "pattern_host_name"

txt2stix/pattern/extractors/domain/hostname_extractor.py

@@ -0,0 +1,36 @@
+from tld import get_tld
+
+from txt2stix.utils import validate_file_mimetype
+from ..helper import TLDs
+
+from ..base_extractor import BaseExtractor
+
+
+class HostnameBaseExtractor(BaseExtractor):
+    """
+    A class for extracting valid hostnames from text using a custom extraction function.
+
+    Attributes:
+        name (str): The name of the extractor, set to "hostname".
+        extraction_regex (function): The custom extraction function to validate and extract hostnames.
+    """
+
+    name = "pattern_host_name"
+    extraction_regex = r'(([\da-zA-Z])([_\w-]{,62})\.){,127}(([\da-zA-Z])[_\w-]{,61})?([\da-zA-Z]\.((xn\-\-[a-zA-Z\d]+)|([a-zA-Z\d]{2,})))'
+
+    @staticmethod
+    def filter_function(domain):
+        """
+        Checks if the given domain is valid based on the number of dots and the top-level domain.
+
+        Args:
+            domain (str): The domain to be checked.
+
+        Returns:
+            bool: True if the domain is valid (has at most 2 dots and a valid TLD), False otherwise.
+        """
+        if domain.count('.') <= 1:
+            tld = get_tld(domain, fix_protocol=True, fail_silently=True)
+            if not tld:
+                return not validate_file_mimetype(domain)
+        return False

txt2stix/pattern/extractors/domain/sub_domain_extractor.py

@@ -0,0 +1,49 @@
+from tld import get_tld
+
+from txt2stix.utils import validate_file_mimetype
+from ..helper import TLDs
+
+from ..base_extractor import BaseExtractor
+
+
+class SubDomainExtractor(BaseExtractor):
+    """
+    A class for extracting valid subdomains from text using a custom extraction function.
+
+    This class inherits from BaseExtractor, which defines the basic structure and functionality for all extractors.
+
+    Attributes:
+        name (str): The name of the extractor, set to "sub-domain".
+        extraction_regex (function): The custom extraction function to validate and extract subdomains.
+    """
+
+    name = "pattern_domain_name_subdomain"
+    extraction_regex = r'(([\da-zA-Z])([_\w-]{,62})\.){,127}(([\da-zA-Z])[_\w-]{,61})?([\da-zA-Z]\.((xn\-\-[' \
+                       r'a-zA-Z\d]+)|([a-zA-Z\d]{2,})))'
+
+    @staticmethod
+    def filter_function(domain):
+        """
+        Checks if the given domain is valid based on the number of dots and the top-level domain.
+
+        Args:
+            domain (str): The domain to be checked.
+
+        Returns:
+            bool: True if the domain is valid (has at most 2 dots and a valid TLD), False otherwise.
+        """
+        if domain.count('.') >= 2:
+            tld = get_tld(domain, fix_protocol=True, fail_silently=True)
+            if tld:
+                domain_name = domain.strip(f".{tld}")
+                if domain_name.count(".") > 0:
+                    return True
+        return False
+
+class HostNameSubdomainExtractor(SubDomainExtractor):
+    name = "pattern_host_name_subdomain"
+    filter_function = lambda domain: domain.count('.') >= 2 and get_tld(domain, fail_silently=True) not in TLDs
+
+    def filter_function(domain):
+        tld =  get_tld(domain, fail_silently=True, fix_protocol=True)
+        return domain.count('.') >= 2 and not tld and not validate_file_mimetype(domain)

txt2stix/pattern/extractors/hashes/__init__.py

@@ -0,0 +1,16 @@
+from .md5_extractor import FileHashMD5Extractor
+from .sha1_extractor import FileHashSHA1Extractor
+from .sha2_256_exactor import FileHashSHA2_256Extractor
+from .sha2_512_exactor import FileHashSHA2_512Extractor
+from .sha3_256_exactor import FileHashSHA3_256Extractor
+from .sha3_512_exactor import FileHashSHA3_512Extractor
+from .sha224_extractor import FileHashSHA224Extractor
+
+SHA_EXTRACTORS = [FileHashMD5Extractor,
+                  FileHashSHA1Extractor,
+                  FileHashSHA224Extractor,
+                  FileHashSHA2_256Extractor,
+                  FileHashSHA2_512Extractor,
+                  FileHashSHA3_256Extractor,
+                  FileHashSHA3_512Extractor
+                  ]

txt2stix/pattern/extractors/hashes/md5_extractor.py

@@ -0,0 +1,16 @@
+import validators
+from ..base_extractor import BaseExtractor
+
+
+class FileHashMD5Extractor(BaseExtractor):
+    """
+    A class for extracting MD5 file hash values from text using a custom extraction function.
+
+    Attributes:
+        name (str): The name of the extractor, set to "md5".
+        extraction_function (function): The custom extraction function to validate and extract MD5 file hash values.
+    """
+
+    name = "pattern_file_hash_md5"
+    extraction_function = lambda x: validators.md5(x)
+

txt2stix/pattern/extractors/hashes/sha1_extractor.py

@@ -0,0 +1,14 @@
+import validators
+from ..base_extractor import BaseExtractor
+
+class FileHashSHA1Extractor(BaseExtractor):
+    """
+    A class for extracting SHA-1 file hash values from text using a custom extraction function.
+
+    Attributes:
+        name (str): The name of the extractor, set to "sha1".
+        extraction_function (function): The custom extraction function to validate and extract SHA-1 file hash values.
+    """
+
+    name = "pattern_file_hash_sha_1"
+    extraction_function = lambda x: validators.sha1(x)

txt2stix/pattern/extractors/hashes/sha224_extractor.py

@@ -0,0 +1,18 @@
+import validators
+from ..base_extractor import BaseExtractor
+
+
+class FileHashSHA224Extractor(BaseExtractor):
+    """
+    A class for extracting SHA-256 file hash values from text using a regular expression.
+
+    Attributes:
+        name (str): The name of the extractor, set to "sha2_256".
+        extraction_regex (str): The regular expression pattern used for extracting SHA-256 file hash values from the text.
+    """
+
+    name = "pattern_file_hash_sha_224"
+    extraction_regex = r'^[0-9a-fA-F]{56}$'
+    extraction_function = lambda x: validators.sha224(x)
+
+# not currently used as not supported by the STIX spec.

txt2stix/pattern/extractors/hashes/sha2_256_exactor.py

@@ -0,0 +1,14 @@
+from ..base_extractor import BaseExtractor
+
+
+class FileHashSHA2_256Extractor(BaseExtractor):
+    """
+    A class for extracting SHA-256 file hash values from text using a regular expression.
+
+    Attributes:
+        name (str): The name of the extractor, set to "sha2_256".
+        extraction_regex (str): The regular expression pattern used for extracting SHA-256 file hash values from the text.
+    """
+
+    name = "pattern_file_hash_sha_256"
+    extraction_regex = r'^[0-9a-fA-F]{64}$'

txt2stix/pattern/extractors/hashes/sha2_512_exactor.py

@@ -0,0 +1,13 @@
+from ..base_extractor import BaseExtractor
+
+class FileHashSHA2_512Extractor(BaseExtractor):
+    """
+    A class for extracting SHA-512 file hash values from text using a regular expression.
+
+    Attributes:
+        name (str): The name of the extractor, set to "sha2_512".
+        extraction_regex (str): The regular expression pattern used for extracting SHA-512 file hash values from the text.
+    """
+
+    name = "pattern_file_hash_sha_512"
+    extraction_regex = r'^[0-9a-fA-F]{128}$'

txt2stix/pattern/extractors/hashes/sha3_256_exactor.py

@@ -0,0 +1,15 @@
+from ..base_extractor import BaseExtractor
+
+
+class FileHashSHA3_256Extractor(BaseExtractor):
+    """
+    A class for extracting SHA-3 (256-bit) file hash values from text using a regular expression.
+
+    Attributes:
+        name (str): The name of the extractor, set to "sha3_256".
+        extraction_regex (str): The regular expression pattern used for extracting SHA-3 (256-bit) file hash values from the text.
+    """
+
+    name = "pattern_file_hash_sha3_256"
+    extraction_regex = r'^[0-9a-fA-F]{64}$'
+

txt2stix/pattern/extractors/hashes/sha3_512_exactor.py

@@ -0,0 +1,16 @@
+from ..base_extractor import BaseExtractor
+
+
+class FileHashSHA3_512Extractor(BaseExtractor):
+    """
+    A class for extracting SHA-3 (512-bit) file hash values from text using a regular expression.
+
+    Attributes:
+        name (str): The name of the extractor, set to "sha3_512".
+        extraction_regex (str): The regular expression pattern used for extracting SHA-3 (512-bit) file hash values from the text.
+    """
+
+    name = "pattern_file_hash_sha3_512"
+    extraction_regex = r'^[0-9a-fA-F]{128}$'
+
+

txt2stix/pattern/extractors/helper.py

@@ -0,0 +1,64 @@
+"""
+Bunch of helper methods
+"""
+import csv
+import logging
+import sys
+
+from .base_extractor import ALL_EXTRACTORS
+
+from ...extractions import Extractor
+from ...utils import FILE_EXTENSIONS, read_included_file, TLDs
+
+def check_false_positive_domain(domain):
+    """
+    Check if a domain is a false positive based on its file extension.
+
+    Args:
+        domain (str): The domain name to be checked.
+
+    Returns:
+        bool: True if the domain is not a false positive, False otherwise.
+    """
+    file_extension = domain.split(".")[-1]
+    if file_extension in FILE_EXTENSIONS:
+        return False
+    else:
+        return True
+
+from txt2stix.utils import validate_file_mimetype as validate_file_extension, validate_tld
+
+def load_extractor(extractor):
+    if extractor.pattern_extractor:
+        return
+    extractor.pattern_extractor = ALL_EXTRACTORS.get(extractor.slug)
+    if not extractor.pattern_extractor:
+        raise TypeError(f"could not find associated python class for pattern extractor `{extractor.slug}`")
+    extractor.pattern_extractor.version = extractor.version
+    extractor.pattern_extractor.stix_mapping = extractor.stix_mapping
+
+
+def extract_all(extractors :list[Extractor], input_text, ignore_extraction_boundary=False):
+    logging.info("using pattern extractors")
+    pattern_extracts = []
+    for extractor in extractors:
+        load_extractor(extractor)
+        extracts = extractor.pattern_extractor().extract_extraction_from_text(input_text)
+        pattern_extracts.extend(extracts)
+
+    pattern_extracts.sort(key=lambda ex: (ex['start_index'], len(ex['value'])))
+    retval = {}
+    end = 0
+    for raw_extract in pattern_extracts:
+        start_index = raw_extract['start_index']
+        key = (raw_extract['type'], raw_extract['value'])
+        if ignore_extraction_boundary or start_index >= end:
+            extraction = retval.setdefault(key, {**raw_extract, "start_index":[start_index]})
+            if start_index not in extraction['start_index']:
+                extraction['start_index'].append(start_index)
+            end = start_index + len(raw_extract['value'])
+    return list(retval.values())
+
+
+# FILE_EXTENSION = read_included_file('lookups/extensions.txt')
+# TLD = read_included_file('lookups/tld.txt')

txt2stix/pattern/extractors/ip/__init__.py

@@ -0,0 +1,14 @@
+from .ipv4_extractor import IPv4Extractor
+from .ipv4_port_extractor import IPv4WithPortExtractor
+from .ipv4_cidr_extractor import IPv4WithCIDRExtractor
+from .ipv6_extractor import IPv6Extractor
+from .ipv6_port_extractor import IPv6WithPortExtractor
+from .ipv6_cidr_extractor import IPv6WithCIDRExtractor
+
+IP_EXTRACTORS = [IPv4Extractor,
+                 IPv4WithPortExtractor,
+                 IPv4WithCIDRExtractor,
+                 IPv6Extractor,
+                 IPv6WithPortExtractor,
+                 IPv6WithCIDRExtractor,
+                 ]

txt2stix/pattern/extractors/ip/ipv4_cidr_extractor.py

@@ -0,0 +1,49 @@
+from ..base_extractor import BaseExtractor
+from ipaddress import IPv4Address
+
+
+class IPv4WithCIDRExtractor(BaseExtractor):
+    """
+    A class for extracting valid IPv4 addresses with CIDR notation from text using a custom extraction function.
+
+    Attributes:
+        name (str): The name of the extractor, set to "ipv4-cidr".
+        extraction_function (function): The custom extraction function to validate and extract IPv4 addresses with CIDR.
+    """
+
+    name = "pattern_ipv4_address_cidr"
+    extraction_function = lambda x: IPv4WithCIDRExtractor.validate_ipv4_with_port(x)
+
+    @staticmethod
+    def validate_ipv4_with_port(x):
+        """
+        Custom extraction function to validate if the provided string is a valid IPv4 address with CIDR.
+
+        Args:
+            x (str): The string to be checked.
+
+        Returns:
+            tuple: A tuple containing the extracted IPv4 address and CIDR if valid, False otherwise.
+        """
+        x = x.strip('"')
+
+        if "https://" in x:
+            x = x.strip("https://")
+
+        if "https://" in x:
+            x = x.strip("http://")
+
+        if "/" in x:
+            ip_address, cidr = x.split("/")
+
+            try:
+                # Validate the IPv4 address part.
+                IPv4Address(ip_address)
+
+                # Validate the CIDR part.
+                if 0 <= int(cidr) <= 32:
+                    return ip_address, cidr
+            except ValueError:
+                pass
+
+        return False

txt2stix/pattern/extractors/ip/ipv4_extractor.py

@@ -0,0 +1,18 @@
+import re
+from ..base_extractor import BaseExtractor
+from ipaddress import IPv4Interface
+import validators
+
+
+class IPv4Extractor(BaseExtractor):
+    """
+    A class for extracting valid IPv4 addresses from text using a custom extraction function and a filter function.
+
+    Attributes:
+        name (str): The name of the extractor, set to "ipv4".
+        extraction_function (function): The custom extraction function to validate and extract IPv4 addresses.
+        filter_function (function): The custom filter function to further filter the extracted IPv4 addresses.
+    """
+
+    name = "pattern_ipv4_address_only"
+    extraction_function = lambda ipaddress: validators.ipv4(ipaddress, strict=True, cidr=False)