opensquat 2.2__0-py3-none-any.whl

opensquat/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+# Module: __init__.py
+"""openSquat Version."""
+__VERSION__ = "2.2.0"

opensquat/app.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# Module: app.py
+"""
+openSquat
+
+(c) Andre Tenreiro
+
+* https://github.com/atenreiro/opensquat
+
+software licensed under GNU version 3
+"""
+import concurrent.futures
+import functools
+import io
+from colorama import Fore, Style
+
+from opensquat import file_input
+from opensquat.feed_manager import FeedManager
+from opensquat.dns_validator import DNSValidator
+from opensquat.squatting_detector import SquattingDetector
+
+
+class Domain:
+    """
+    Main orchestration class for OpenSquat.
+    """
+
+    def __init__(self):
+        """Initiator."""
+        self.domain_filename = None
+        self.keywords_filename = None
+        self.domain_total = 0
+        self.keywords_total = 0
+        self.list_domains = []
+        self.confidence_level = 2
+        self.doppelganger_only = False
+
+        self.feed_manager = None
+        self.dns_validator = None
+        self.squatting_detector = None
+
+        self.list_file_domains = []
+        self.list_file_keywords = []
+
+    def count_files(self):
+        (self.keywords_total, self.domain_total) = file_input.InputFile().main(
+            self.keywords_filename,
+            self.domain_filename
+        )
+
+    def read_files(self):
+        """
+        Method to read domain files
+        """
+        with open(self.domain_filename, mode='r') as file_domains:
+            for mydomains in file_domains:
+                domain = mydomains.replace("\n", "")
+                domain = domain.lower().strip()
+                # Skip comments and empty lines
+                if domain and not domain.startswith("#"):
+                    self.list_file_domains.append(domain)
+
+        with open(self.keywords_filename, mode='r') as file_keywords:
+            for line in file_keywords:
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    self.list_file_keywords.append(line)
+
+    def print_info(self):
+        """
+        Method to print some configuration information.
+        """
+        print("[*] keywords:", self.keywords_filename)
+        print("[*] keywords total:", self.keywords_total)
+        print("[*] Total domains:", f"{self.domain_total:,}")
+
+        print("[*] Threshold:", self.squatting_detector.confidence.get(self.confidence_level, "unknown"))
+
+    @staticmethod
+    def verify_keyword_task(detector, domains_list, keyword_info):
+        """
+        Static worker method for parallel execution.
+        """
+        keyword, keyword_line_number, keywords_total = keyword_info
+
+        result_buffer = io.StringIO()
+        print(
+            f"[+] Starting Domain Squatting verification for '{keyword}' [{keyword_line_number}/{keywords_total}]",
+            file=result_buffer
+        )
+
+        print(
+            Fore.WHITE + "\n[*] Verifying keyword:",
+            keyword,
+            "[",
+            keyword_line_number,
+            "/",
+            keywords_total,
+            "]" + Style.RESET_ALL,
+            file=result_buffer
+        )
+
+        result_domains = detector.check(keyword, domains_list, result_buffer)
+        return result_buffer, result_domains
+
+    def worker(self):
+        """
+        Method that will compute all the similarity calculations between
+        the keywords and domain names.
+        """
+        with concurrent.futures.ProcessPoolExecutor() as executor:
+            keyword_infos = [
+                (keyword, i + 1, self.keywords_total)
+                for i, keyword in enumerate(self.list_file_keywords) if keyword
+            ]
+
+            worker_func = functools.partial(self.verify_keyword_task, self.squatting_detector, self.list_file_domains)
+
+            futs = [executor.submit(worker_func, k_info) for k_info in keyword_infos]
+
+        for fut in futs:
+            result_buffer, result_domains = fut.result()
+            print(result_buffer.getvalue())
+            self.list_domains.extend(result_domains)
+
+        return self.list_domains
+
+    def main(
+        self,
+        keywords_file,
+        confidence_level,
+        domains_file,
+        dns,
+        doppelganger_only=False,
+        feed_url="https://feeds.opensquat.com/opensquat-nrd-latest.txt",
+    ):
+        print("+---------- Checking Domain Squatting ----------+")
+        self.keywords_filename = keywords_file
+        self.domain_filename = domains_file
+        self.confidence_level = confidence_level
+        self.doppelganger_only = doppelganger_only
+
+        self.dns_validator = DNSValidator(use_dns=dns)
+        self.squatting_detector = SquattingDetector(
+            confidence_level=confidence_level,
+            doppelganger_only=doppelganger_only,
+            dns_validator=self.dns_validator
+        )
+
+        # Feed Management
+        self.feed_manager = FeedManager(feed_url=feed_url)
+        if self.domain_filename == "":
+            self.domain_filename = self.feed_manager.local_filename
+            self.feed_manager.ensure_feeds()
+
+        self.count_files()
+        self.read_files()
+        self.print_info()
+
+        return self.worker()

opensquat/arg_parser.py

@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+# Module: arg_parser.py
+"""
+openSquat.
+
+(c) Andre Tenreiro
+
+* https://github.com/atenreiro/opensquat
+
+software licensed under GNU version 3
+
+"""
+import argparse
+
+
+def validate_type(file_type):
+    """
+    Validate file_type.
+
+    Args:
+        file_type: string containing file type, can only be txt, json or csv.
+
+    Return:
+        file_type
+
+    Raise:
+        If value is not valid, raise an exception to argparse
+    """
+    file_type = str(file_type)
+
+    if (file_type != "txt") and (file_type != "json") and (file_type != "csv"):
+        raise argparse.ArgumentTypeError("File format unkown!")
+    return file_type
+
+
+def validate_confidence(confidence_level):
+    """
+    Validate confidence_level.
+
+    Args:
+        confidence_level: int containing confidence_level, can only be an int
+        between 0 and 4.
+
+    Return:
+        confidence_level
+
+    Raise:
+        If value is not valid, raise an exception to argparse
+
+    """
+    confidence_level = int(confidence_level)
+
+    if confidence_level not in range(0, 5):
+        raise argparse.ArgumentTypeError("confidence must be between 0 and 4")
+    return confidence_level
+
+
+def get_args():
+    """
+    Parser main function.
+
+    Args:
+        none
+
+    Return:
+        args: returns arguments
+    """
+    parser = argparse.ArgumentParser(description="openSquat")
+    parser.add_argument(
+        "-k",
+        "--keywords",
+        type=str,
+        default="keywords.txt",
+        help="keywords file (default: keywords.txt)",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        default="results.txt",
+        help="output filename (default: results.txt)",
+    )
+    parser.add_argument(
+        "-c",
+        "--confidence",
+        type=validate_confidence,
+        default=1,
+        help="0 (very high), 1 (high), 2 (medium), 3 (low),"
+        "4 (very low) (default: 1)",
+    )
+    parser.add_argument(
+        "-t",
+        "--type",
+        type=validate_type,
+        default="txt",
+        help="output file type [txt|json|csv] (default: txt)",
+    )
+    parser.add_argument(
+        "-d",
+        "--domains",
+        type=str,
+        default="",
+        help="update from FILE instead of downloading new domains",
+    )
+    parser.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        default="https://feeds.opensquat.com/opensquat-nrd-latest.txt",
+        help="URL to download domain feed (default: https://feeds.opensquat.com/opensquat-nrd-latest.txt)",
+    )
+    parser.add_argument(
+        "-p",
+        "--period",
+        type=str,
+        default=None,
+        help=argparse.SUPPRESS,  # Hide from help since deprecated
+    )
+
+    parser.add_argument(
+        "--ct",
+        action="store_true",
+        help="search in certificate transparency",
+    )
+
+    parser.add_argument(
+        "--doppelganger",
+        action="store_true",
+        help="doppelganger-only mode: check if domains contain the keyword and are reachable",
+    )
+
+    parser.add_argument(
+        "--dns",
+        action="store_true",
+        help="Check if domain is flagged by Quad9 DNS"
+    )
+
+    parser.add_argument(
+        "--phishing",
+        type=str,
+        default="",
+        help="search known and active Phishing sites (arg: output.txt)",
+    )
+    parser.add_argument(
+        "--subdomains",
+        action="store_true",
+        help="search for subdomains from flagged domains",
+    )
+    parser.add_argument(
+        "--portcheck",
+        action="store_true",
+        help="Verify is port 80/443 is open",
+    )
+    parser.add_argument(
+        "--vt",
+        action="store_true",
+        help="validate against VirusTotal",
+    )
+
+    args = parser.parse_args()
+
+    # Check for deprecated -p/--period argument
+    if args.period is not None:
+        print("\n[ERROR] The weekly/monthly feeds have been deprecated. Please use the daily feeds.\n")
+        exit(1)
+
+    return args

opensquat/check_update.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Module: check_update.py
+"""
+openSquat
+
+* https://github.com/atenreiro/opensquat
+
+software licensed under GNU version 3
+"""
+import requests
+from colorama import Fore, Style
+from opensquat import __VERSION__
+
+class CheckUpdate:
+    """
+    This domain class verifies if there is a newer upgrade avaiable
+
+    To use:
+        CheckUpdate.main()
+
+    Attribute:
+        URL: The URL with the latest upgrade version
+        current: The current running version
+    """
+
+    def __init__(self):
+        """Initiator."""
+        self.URL = ("https://feeds.opensquat.com/latest_version.txt")
+        self.current = __VERSION__
+
+    def check(self):
+
+        # User-Agent
+        ver = "openSquat-" + self.current
+        headers = {'User-Agent': ver}
+
+        try:
+            response = requests.get(self.URL, headers=headers)
+        except requests.exceptions.RequestException:
+            return False
+
+        if (response.status_code != 200):
+            return False
+
+        latest_ver = response.content
+        latest_ver = latest_ver.decode()
+        response.close()
+
+        current_ver = self.current
+
+        def parse_version(v):
+            return tuple(int(x) for x in v.strip().split("."))
+
+        if parse_version(latest_ver) > parse_version(current_ver):
+            print(
+                Style.BRIGHT+Fore.MAGENTA +
+                "[INFO] New version avaiable!" +
+                Style.RESET_ALL)
+            print(
+                Style.BRIGHT+Fore.WHITE +
+                "-> Current ver:", current_ver, Style.RESET_ALL)
+            print(
+                Style.BRIGHT+Fore.WHITE +
+                "-> Latest ver:", latest_ver, Style.RESET_ALL)
+            print(
+                Style.BRIGHT+Fore.WHITE +
+                "-> Changelog: https://github.com/atenreiro/opensquat/blob/" +
+                "master/CHANGELOG" +
+                Style.RESET_ALL)
+            print(
+                Style.BRIGHT+Fore.WHITE + "-> update now: $ git pull\n" +
+                Style.RESET_ALL)
+            return True
+
+        return False
+
+    def main(self):
+        self.check()

opensquat/cli.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Module: cli.py
+"""
+openSquat CLI entry point.
+
+* https://github.com/atenreiro/opensquat
+
+software licensed under GNU version 3
+"""
+import time
+import signal
+import functools
+import concurrent.futures
+
+from colorama import init, Fore, Style
+from opensquat import __VERSION__, vt
+from opensquat import arg_parser, output, app, phishing, check_update
+from opensquat import port_check
+
+
+def signal_handler(sig, frame):
+    """Function to catch CTR+C and terminate."""
+    print("\n[*] openSquat is terminating...\n")
+    exit(0)
+
+
+def main():
+    signal.signal(signal.SIGINT, signal_handler)
+
+    init()
+
+    logo = (
+        Style.BRIGHT + Fore.GREEN +
+        """
+                                             █████████                                  █████
+                                            ███░░░░░███                                ░░███
+      ██████  ████████   ██████  ████████  ░███    ░░░   ████████ █████ ████  ██████   ███████
+     ███░░███░░███░░███ ███░░███░░███░░███ ░░█████████  ███░░███ ░░███ ░███  ░░░░░███ ░░░███░
+    ░███ ░███ ░███ ░███░███████  ░███ ░███  ░░░░░░░░███░███ ░███  ░███ ░███   ███████   ░███
+    ░███ ░███ ░███ ░███░███░░░   ░███ ░███  ███    ░███░███ ░███  ░███ ░███  ███░░███   ░███ ███
+    ░░██████  ░███████ ░░██████  ████ █████░░█████████ ░░███████  ░░████████░░████████  ░░█████
+     ░░░░░░   ░███░░░   ░░░░░░  ░░░░ ░░░░░  ░░░░░░░░░   ░░░░░███   ░░░░░░░░  ░░░░░░░░    ░░░░░
+              ░███                                          ░███
+              █████                                         █████
+             ░░░░░                                         ░░░░░
+                    (c) openSquat - https://opensquat.com
+    """ + Style.RESET_ALL
+    )
+
+    print(logo)
+    print("\t\t\tversion " + __VERSION__ + "\n")
+
+    args = arg_parser.get_args()
+
+    start_time_squatting = time.time()
+
+    file_content = app.Domain().main(
+        args.keywords,
+        args.confidence,
+        args.domains,
+        args.dns,
+        doppelganger_only=args.doppelganger,
+        feed_url=args.url,
+    )
+
+    if args.subdomains or args.vt or args.phishing or args.portcheck:
+        print("\n[*] Total found:", len(file_content))
+
+    # Check for subdomains
+    if (args.subdomains):
+        list_aux = []
+        print("\n+---------- Checking for Subdomains ----------+")
+        time.sleep(1)
+        for domain in file_content:
+            print("[*]", domain)
+            subdomains = vt.VirusTotal().main(domain, "subdomains")
+
+            if subdomains:
+                for subdomain in subdomains:
+                    print(
+                        Style.BRIGHT + Fore.YELLOW +
+                        " \\_", subdomain +
+                        Style.RESET_ALL,
+                        )
+                    list_aux.append(subdomain)
+        file_content = list_aux
+        print("[*] Total found:", len(file_content))
+
+    # Check for VirusTotal (if domain is flagged as malicious)
+    if (args.vt):
+        list_aux = []
+        print("\n+---------- VirusTotal ----------+")
+        time.sleep(1)
+        for domain in file_content:
+            total_votes = vt.VirusTotal().main(domain)
+
+            # total votes
+            harmless = total_votes[0]
+            malicious = total_votes[1]
+
+            if malicious > 0:
+                print(
+                    Style.BRIGHT + Fore.RED +
+                    "[*] found:", domain, "({})".format(str(malicious)) +
+                    Style.RESET_ALL,
+                    )
+                list_aux.append(domain)
+            elif malicious < 0:
+                print(
+                    Style.BRIGHT + Fore.YELLOW +
+                    "[*] VT is throttling the response:", domain +
+                    Style.RESET_ALL,
+                    )
+                list_aux.append(domain)
+        file_content = list_aux
+        print("[*] Total found:", len(file_content))
+
+    # Check for phishing
+    if (args.phishing != ""):
+        file_phishing = phishing.Phishing().main(args.keywords)
+        output.SaveFile().main(args.phishing, "txt", file_phishing)
+
+    # Check if domain has webserver port opened
+    if (args.portcheck):
+        list_aux = []
+        print("\n+---------- Domains with open webserver ports ----------+")
+        time.sleep(1)
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futs = [(domain, executor.submit(functools.partial(port_check.PortCheck().main, domain)))
+                    for domain in file_content]
+
+        for tested_domain, result_domain_port_check in futs:
+            ports = result_domain_port_check.result()
+            if ports:
+                list_aux.append(tested_domain)
+                print(
+                    Fore.YELLOW +
+                    "[*]", tested_domain, ports, "" +
+                    Style.RESET_ALL
+                    )
+
+        file_content = list_aux
+        print("[*] Total found:", len(file_content))
+
+    output.SaveFile().main(args.output, args.type, file_content)
+    end_time_squatting = round(time.time() - start_time_squatting, 2)
+
+    # Print summary
+    print("\n")
+    print(
+        Style.BRIGHT+Fore.GREEN +
+        "+---------- Summary Squatting ----------+" +
+        Style.RESET_ALL)
+
+    print("[*] Domains flagged:", len(file_content))
+    print("[*] Domains result:", args.output)
+
+    if (args.phishing != ""):
+        print("[*] Phishing results:", args.phishing)
+        print("[*] Active Phishing sites:", len(file_phishing))
+
+    print("[*] Running time: %s seconds" % end_time_squatting)
+    print("")
+
+    check_update.CheckUpdate().main()
+
+
+if __name__ == "__main__":
+    main()

opensquat/ct.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Module: ct.py
+"""
+openSquat.
+
+(c) Andre Tenreiro
+
+* https://github.com/atenreiro/opensquat
+
+software licensed under GNU version 3
+"""
+import requests
+from bs4 import BeautifulSoup, NavigableString
+
+NOT_TRUSTED_CA = ["Let's Encrypt Authority X3"]
+
+
+class CTLog:
+    def __init__(
+        self, _id, logget_at, not_before, not_after, matching_ident, issuer_name
+    ):
+        self._id = _id
+        self.logget_at = logget_at
+        self.not_before = not_before
+        self.not_after = not_after
+        self.matching_ident = matching_ident
+        self.issuer_name = issuer_name
+
+
+class CRTSH:
+    """Class responsible for checking given domain for CT logs."""
+
+    URL = "https://crt.sh/"
+
+    @classmethod
+    def check_certificate(cls, domain: str) -> bool:
+        url = f"{cls.URL}?q={domain}"
+
+        try:
+            html_text = requests.get(url).text
+        except Exception:
+            print(f"Cannot fetch data from {url}")
+            return True
+
+        soup = BeautifulSoup(html_text, "html.parser")
+
+        collected_logs = []
+        for table in soup.find_all("table")[1]:
+            if isinstance(table, NavigableString):
+                continue
+
+            for table_row in table.find_all("tr")[1:]:
+                tds = [td for td in table_row.find_all("td")]
+                try:
+                    collected_logs.append(
+                        CTLog(
+                            tds[0].text,
+                            tds[1].text,
+                            tds[2].text,
+                            tds[3].text,
+                            [x for x in tds[4] if isinstance(x, NavigableString)],
+                            tds[5].text,
+                        )
+                    )
+                except Exception as e:
+                    print(f"Could not retreive CT logs from {domain}, {e}")
+                    return False
+
+        for ca in NOT_TRUSTED_CA:
+            for ctlog in collected_logs:
+                if domain in ctlog.matching_ident:  # checking domain matching
+                    if ca in ctlog.issuer_name:  # checking CA
+                        return False
+
+        if not collected_logs:
+            return False
+
+        return True