subprober 3.1.0__py3-none-any.whl

subprober/__init__.py

@@ -0,0 +1,14 @@
+from subprober.httpclient.httpclient import RetryableHttp, HttpResponse
+from subprober.pyrunner.pyrunner import PyRunner
+from subprober.settings.settings import Settings
+from subprober.hmap.hmap import HMap
+from subprober.workerpool.workerpool import WorkerPool
+
+__all__ = [
+    "RetryableHttp",
+    "HttpResponse",
+    "PyRunner",
+    "Settings",
+    "HMap",
+    "WorkerPool",
+]

subprober/cli/cli.py

@@ -0,0 +1,209 @@
+from richparser import RichParser
+
+class CLI:
+    def __init__(self):
+        self.parser = None
+
+    def cli(self):
+        try:
+            parser = RichParser(
+                description="Subprober - An essential HTTP multi-purpose Probing Tool for Penetration Testers and Security Researchers with Asynchronous httpx client support ⚡",
+                auto=False
+            )
+
+            # INPUT Section
+            parser.add_argument("input", "-l", "--list", type=str,
+                                help="specify the filename containing a list of URLs to probe")
+            parser.add_argument("input", "-u", "--url", type=str,
+                                help="specify a URL to probe and supports comma-separated values (-u google.com,https://hackerone.com)")
+            parser.add_argument("input", "-resume", "--resume", type=str, default=None,
+                                help="specify the resume filename generated by the subprober to continue the scan (-resume resume_Djjfos.cfg)")
+
+            # PROBES Section
+            parser.add_argument("probes", "-status-code", "--status-code", action="store_true",
+                                help="display the status code of the host")
+            parser.add_argument("probes", "-title", "--title", action="store_true", help="display the title of host")
+            parser.add_argument("probes", "-server", "--server", action="store_true",
+                                help="display the server name of the host")
+            parser.add_argument("probes", "-wc", "--word-count", action="store_true",
+                                help="display the HTTP response word count")
+            parser.add_argument("probes", "-lc", "--line-count", action="store_true",
+                                help="display the HTTP response line count")
+            parser.add_argument("probes", "-cl", "--content-length", action="store_true",
+                                help="display the HTTP response content length")
+            parser.add_argument("probes", "-location", "--location", action="store_true",
+                                help="display the redirected location of the host")
+            parser.add_argument("probes", "-application-type", "--application-type", action="store_true",
+                                help="display the content type of the host")
+            parser.add_argument("probes", "-ip", "--ipaddress", action="store_true", help="display the IPs of the host")
+            parser.add_argument("probes", "-cname", "--cname", action="store_true", help="display the CNAMEs of the host")
+            parser.add_argument("probes", "-aaaa", "--aaaa-records", action="store_true",
+                                help="display the AAAA records of the host")
+            parser.add_argument("probes", "-htv", "--http-version", action="store_true",
+                                help="display the server supported HTTP version of the host")
+            parser.add_argument("probes", "-hrs", "--http-reason", action="store_true",
+                                help="display the reason for HTTP connection of the host")
+            parser.add_argument("probes", "-jarm", "--jarm-fingerprint", action="store_true",
+                                help="display the JARM fingerprint hash of the host")
+            parser.add_argument("probes", "-rpt", "--response-time", action="store_true",
+                                help="display the response time for the successful request")
+            parser.add_argument("probes", "-wss", "--websocket", action="store_true",
+                                help="display the server supports websockets")
+            parser.add_argument("probes", "-hash", "--hash", type=str,
+                                help="display response body in hash format (supported hashes: md5, mmh3, simhash, sha1, sha256, sha512)")
+            parser.add_argument("probes", "-dmt", "--display-method", action="store_true",
+                                help="display the method of the HTTP request")
+            parser.add_argument("probes", "-bp", "--body-preview", action="store_true",
+                                help="display the HTTP response body in first n number of characters (default: 100)")
+            parser.add_argument("probes", "-body", "--body", type=str, default=None,
+                                help="post body to include in the http request and support all post body types (ex: -body 'username=admin&password=password')")
+            parser.add_argument("probes", "-resolvers", "--resolvers", type=str, default="8.8.8.8,1.1.1.1",
+                                help="custom DNS resolver for dns resolution and supports comma-separated (ex -resolvers 8.8.8.8,1.1.1.1)")
+
+            # CONFIG Section
+            parser.add_argument("config", "-dhp", "--disable-http-probe", action="store_true",
+                                help="disable subprober from fallback to http scheme (default: disabled)")
+            parser.add_argument("config", "-X", "--method", type=str,
+                                choices=["get", "post", "head", "put", "delete", "patch", "trace", "connect",
+                                         "options"], default="get",
+                                help="request methods to probe and get response (supported: get, post, head, put, delete, patch, trace, connect, options) (default: get)")
+            parser.add_argument("config", "-H", "--header", action="append",
+                                help="add custom headers for probing and -H can be used multiple times to pass multiple header values (ex: -H application/json -H X-Forwarded-Host: 127.0.0.1)")
+            parser.add_argument("config", "-ra", "--random-agent", action="store_true",
+                                help="enable Random User-Agent to use for probing and applies same to screenshots. (default: subprober/Alpha)")
+            parser.add_argument("config", "-proxy", "--proxy", type=str,
+                                help="specify a proxy to send the requests through it (ex: http://127.0.0.1:8080)", default=None)
+            parser.add_argument("config", "-ar", "--allow-redirect", action="store_true",
+                                help="enable following redirections")
+            parser.add_argument("config", "-maxr", "--max-redirection", type=int, default=10,
+                                help="set max value to follow redirections (default: 10)")
+            parser.add_argument("config", "-http2", "--http2", action="store_true",
+                                help="enable to request with HTTP/2 support (default: Http/1.1) (info: deprecated)")
+            parser.add_argument("config", "-sni", "--sni-hostname", type=str,
+                                help="set custom TLS SNI host name for requests.")
+
+            parser.add_argument("config", "-stats", "--stats",action="store_true",
+                                help="show the progress stats of the subprober (info: specially for docker environments)")
+
+            # MISCELLANEOUS Section
+            parser.add_argument("miscellaneous", "-path", "--path", type=str,
+                                help="specify a path or text file of paths for probing and getting results (example: -p admin.php or -p paths.txt)")
+            parser.add_argument("miscellaneous", "-port", "--port", type=str,
+                                help="set custom port for making HTTP request and default ports are 80,443 based on the url scheme")
+            parser.add_argument("miscellaneous", "-tls", "--tls", action="store_true",
+                                help="grabs the TLS data for the requested host")
+
+            # HEADLESS Section
+            parser.add_argument("headless", "-ss", "--screenshot", action="store_true",
+                                help="enable to take screenshots of the page using headless browsers with asynchronous performance")
+            parser.add_argument("headless", "-st", "--screenshot-timeout", type=int, default=10,
+                                help="set a timeout value for taking screenshots (default: 15)")
+            parser.add_argument("headless", "-scp", "--system-chrome-path", type=str, default=None,
+                                help="specify the executable path of the chromedriver to use system chrome to take screenshots")
+            parser.add_argument("headless", "-pdf", "--save-pdf", action="store_true",
+                                help="enable to save the screenshot image in the pdf format (default: png)")
+            parser.add_argument("headless", "-no-fpg", "--no-full-page", action="store_true",
+                                help="disable saving screenshot in full page")
+            parser.add_argument("headless", "-icb", "--include-bytes", action="store_true",
+                                help="enable to include the screenshot bytes in output when json output enabled")
+            parser.add_argument("headless", "-hos", "--headless-options", type=str, default=None,
+                                help="set additional chrome headless browser options and supports comma-separated values (-ho \"--start-maximized\")")
+            parser.add_argument("headless", "-sid", "--screenshot-idle", default=1, type=int,
+                                help="set custom idle time in seconds before taking screenshots (default: 1)")
+            parser.add_argument("headless", "-sp", "--screenshot-path", type=str,
+                                help="specify a directory path to store screenshot results (default: currentdir/screenshots)")
+
+            # MATCHERS Section
+            parser.add_argument("matchers", "-mc", "--match-code", type=str,
+                                help="match http response by specified status codes and supports comma-separated values (-mc 200,302)")
+            parser.add_argument("matchers", "-mcr", "--match-code-range", type=str,
+                                help="match http response by specified status code range and supports single value (-mcr 200-299)")
+            parser.add_argument("matchers", "-ms", "--match-string", type=str,
+                                help="match http response containing the specified string and supports comma-separated values (-ms admin,login)")
+            parser.add_argument("matchers", "-mr", "--match-regex", type=str,
+                                help="match http response matching the specified regex and supports comma-separated values (-mr .*admin.*,.*login.*)")
+            parser.add_argument("matchers", "-mpt", "--match-path", type=str,
+                                help="match http response by URL path and supports comma-separated values (-mpt /admin/wp-ajax.php,/wp-json)")
+            parser.add_argument("matchers", "-ml", "--match-length", type=str,
+                                help="match http response by specified response length and supports comma-separated values (-ml 1024,2048)")
+            parser.add_argument("matchers", "-mlc", "--match-line-count", type=str,
+                                help="match http response by specified response line count and supports comma-separated values (-mlc 10,50)")
+            parser.add_argument("matchers", "-mwc", "--match-word-count", type=str,
+                                help="match http response by specified word count and supports comma-separated values (-mwc 100,500)")
+            parser.add_argument("matchers", "-mrt", "--match-response-time", type=float,
+                                help="match http response exceeding the specified minimum response time in seconds (-mrt 2.30)")
+
+            # FILTERS Section
+            parser.add_argument("filters", "-fc", "--filter-code", type=str,
+                                help="filter http response by specified status codes and supports comma-separated values (-fc 404,500)")
+            parser.add_argument("filters", "-fcr", "--filter-code-range", type=str,
+                                help="filter http response by specified status code range and supports single value (-fcr 400-499)")
+            parser.add_argument("filters", "-fs", "--filter-string", type=str,
+                                help="filter http response containing the specified string and supports comma-separated values (-fs error,not found)")
+            parser.add_argument("filters", "-fr", "--filter-regex", type=str,
+                                help="filter http response matching the specified regex and supports comma-separated values (-fr .*admin.*,.*login.*)")
+            parser.add_argument("filters", "-fpt", "--filter-path", type=str,
+                                help="filter http response by URL path and supports comma-separated values (-fpt /error,404.html)")
+            parser.add_argument("filters", "-fl", "--filter-length", type=str,
+                                help="filter http response by specified response length and supports comma-separated values (-fl 1024,2048)")
+            parser.add_argument("filters", "-flc", "--filter-line-count", type=str,
+                                help="filter http response by specified response line count and supports comma-separated values (-flc 10,50)")
+            parser.add_argument("filters", "-fwc", "--filter-word-count", type=str,
+                                help="filter http response by specified response word count and supports comma-separated values (-fwc 100,500)")
+            parser.add_argument("filters", "-frt", "--filter-response-time", type=float,
+                                help="filter http response exceeding the specified maximum response time in seconds (-frt 2.30)")
+
+            # OUTPUT Section
+            parser.add_argument("output", "-o", "--output", type=str,
+                                help="define the output filename to store the results of the probing operation.")
+            parser.add_argument("output", "-json", "--json", action="store_true",
+                                help="store and display output in JSON format (includes only data from enabled options).")
+            parser.add_argument("output", "-rdu", "--redirect-urls", action="store_true",
+                                help="display the redirect URLs in the output (requires -json and -ar to enabled to enabled).")
+            parser.add_argument("output", "-rdh", "--redirect-history", action="store_true",
+                                help="display the full redirect history (requires -json and -ar to enabled).")
+            parser.add_argument("output", "-rsc", "--redirect-status-codes", action="store_true",
+                                help="display the status codes for redirections (requires -json and -ar to enabled).")
+            parser.add_argument("output", "-rqh", "--request-headers", action="store_true",
+                                help="include request headers in the output (requires -json and -ar to enabled).")
+            parser.add_argument("output", "-rsh", "--response-headers", action="store_true",
+                                help="include response headers in the output (requires -json and -ar to enabled).")
+            parser.add_argument("output", "-fo", "--full-output", action="store_true",
+                                help="include all available data in the output (requires -json to enabled and doesn't overrides websocket,jarm,hashes options).")
+
+            # RATE-LIMIT Section
+            parser.add_argument("rate-limit", "-c", "--concurrency", type=int, default=100,
+                                help="set the concurrency level for sending http requests (default: 100)")
+            parser.add_argument("rate-limit", "-rtl", "--rate-limit", type=int, default=150,
+                                help="set a rate limit for sending a maximum number of requests per second (default: 1000)")
+
+            # OPTIMIZATION Section
+            parser.add_argument("optimization", "-timeout", "--timeout", type=int, default=10,
+                                help="set a custom timeout value for sending requests.")
+            parser.add_argument("optimization", "-delay", "--delay", type=float, default=0.5,
+                                help="set a delay in seconds before sending each request (default: None)")
+            parser.add_argument("optimization", "-rts", "--retries", type=int, default=0,
+                                help="set a number of retries if a request fails to connect (default: 0)")
+
+            # UPDATES Section
+            parser.add_argument("updates", "-up", "--update", action="store_true",
+                                help="update subprober to the latest version (uv tool required to be installed)")
+            parser.add_argument("updates", "-sup", "--show-updates", action="store_true",
+                                help="display the current or latest version of subprober updates")
+
+
+            parser.add_argument("debug", "-silent", "--silent", action="store_true",
+                                help="enable silent mode to suppress the display of Subprober banner and version information.")
+            parser.add_argument("debug", "-verbose", "--verbose", action="store_true",
+                                help="enable verbose mode to display error results on the console.")
+            parser.add_argument("debug", "-nc", "--no-color", action="store_true",
+                                help="enable to display the output without any CLI colors")
+            parser.add_argument("debug", "-debug", "--debug", action="store_true", help="debug the subprober workflow")
+
+            self.parser = parser
+            return parser.parse_args()
+        except KeyboardInterrupt:
+            exit(1)
+        except Exception as e:
+            print(f"Exception occurred in the cli module due to: {e}, {type(e)}")
+            exit(1)

subprober/dnsclient/dnsclient.py

@@ -0,0 +1,321 @@
+import aiodns
+import asyncio
+import socket
+from typing import Optional, List
+from contextlib import asynccontextmanager
+
+
+class AsyncDnsClient:
+    """
+    Asynchronous DNS client for domain resolution.
+
+    This client uses aiodns library to perform DNS queries with retry logic.
+    Concurrency control is handled by the caller.
+    """
+
+    def __init__(
+            self,
+            nameservers: Optional[List[str]] = None,
+            max_retries: int = 3,
+            timeout: float = 5.0,
+            verbose: bool = False
+    ):
+        """
+        Initialize DNS client.
+
+        Args:
+            nameservers: List of DNS servers to use (default: Cloudflare and Google)
+            max_retries: Maximum retry attempts per query
+            timeout: Query timeout in seconds
+            verbose: Enable verbose logging
+        """
+        self.nameservers = nameservers or ["1.1.1.1", "8.8.8.8"]
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self.verbose = verbose
+        self._resolver = None
+        self._resolver_lock = asyncio.Lock()
+
+    def _log(self, message: str, level: str = "INFO"):
+        """Log message if verbose is enabled."""
+        if self.verbose:
+            print(f"[DNS-{level}] {message}")
+
+    async def _get_resolver(self) -> aiodns.DNSResolver:
+        """
+        Get or create a shared DNS resolver instance.
+
+        Reuses a single resolver instance to avoid exhausting system inotify watches.
+        Thread-safe through async lock.
+        """
+        if self._resolver is None:
+            async with self._resolver_lock:
+                # Double-check after acquiring lock
+                if self._resolver is None:
+                    try:
+                        loop = asyncio.get_running_loop()
+                        self._resolver = aiodns.DNSResolver(
+                            loop=loop,
+                            nameservers=self.nameservers,
+                            rotate=True,
+                            timeout=self.timeout
+                        )
+                        self._log(f"DNS resolver created with nameservers: {self.nameservers}")
+                    except Exception as e:
+                        self._log(f"Failed to create DNS resolver: {e}", "ERROR")
+                        raise
+        return self._resolver
+
+    def _extract_domain(self, url: str) -> Optional[str]:
+        """
+        Extract domain from URL.
+
+        Args:
+            url: URL or domain string
+
+        Returns:
+            Cleaned domain name or None if invalid
+        """
+        if not url:
+            return None
+
+        # Remove protocol
+        if "://" in url:
+            url = url.split("://", 1)[1]
+
+        # Remove path, query, fragment
+        url = url.split("/", 1)[0].split("?", 1)[0].split("#", 1)[0]
+
+        # Remove port
+        url = url.split(":", 1)[0]
+
+        # Remove trailing dot
+        url = url.rstrip(".")
+
+        return url.lower() if url else None
+
+    async def _query_with_retry(
+            self,
+            domain: str,
+            record_type: str,
+            resolver: aiodns.DNSResolver
+    ) -> List[str]:
+        """
+        Execute DNS query with retry logic.
+
+        Args:
+            domain: Domain to query
+            record_type: DNS record type (A, AAAA, CNAME, etc.)
+            resolver: DNS resolver instance
+
+        Returns:
+            List of resolved records
+        """
+        last_exception = None
+
+        for attempt in range(1, self.max_retries + 1):
+            try:
+                self._log(f"Querying {domain} ({record_type}) - Attempt {attempt}/{self.max_retries}")
+
+                results = await asyncio.wait_for(
+                    resolver.query(domain, record_type),
+                    timeout=self.timeout
+                )
+
+                # Parse results based on record type
+                if record_type == 'CNAME':
+                    records = [results.cname] if hasattr(results, 'cname') else []
+                elif record_type in ('A', 'AAAA'):
+                    records = [result.host for result in results] if results else []
+                elif record_type == 'MX':
+                    records = [result.host for result in results] if results else []
+                elif record_type == 'TXT':
+                    records = [result.text for result in results] if results else []
+                elif record_type == 'NS':
+                    records = [result.host for result in results] if results else []
+                elif record_type == 'SOA':
+                    if results:
+                        soa = results
+                        records = [f"{soa.nsname} {soa.hostmaster}"]
+                    else:
+                        records = []
+                elif record_type == 'PTR':
+                    records = [result.host for result in results] if results else []
+                else:
+                    records = []
+
+                if records:
+                    self._log(f"Successfully resolved {domain} ({record_type}): {records}")
+                    return records
+                else:
+                    self._log(f"No records found for {domain} ({record_type})")
+                    return []
+
+            except asyncio.TimeoutError:
+                last_exception = f"Timeout after {self.timeout}s"
+                self._log(f"Timeout for {domain} ({record_type}) on attempt {attempt}", "WARN")
+
+            except aiodns.error.DNSError as e:
+                # DNS-specific errors (NXDOMAIN, SERVFAIL, etc.)
+                if e.args[0] == aiodns.error.ARES_ENOTFOUND:
+                    self._log(f"Domain not found: {domain}", "DEBUG")
+                    return []  # No retry for NXDOMAIN
+                elif e.args[0] == aiodns.error.ARES_ENODATA:
+                    self._log(f"No data for {domain} ({record_type})", "DEBUG")
+                    return []  # No retry for NODATA
+                last_exception = f"DNS error: {e}"
+                self._log(f"DNS error for {domain} ({record_type}): {e}", "WARN")
+
+            except socket.gaierror as e:
+                last_exception = f"Socket error: {e}"
+                self._log(f"Socket error for {domain} ({record_type}): {e}", "WARN")
+
+            except (KeyboardInterrupt, asyncio.CancelledError):
+                self._log("Query cancelled by user", "INFO")
+                raise
+
+            except Exception as e:
+                last_exception = f"Unexpected error: {e}"
+                self._log(f"Unexpected error for {domain} ({record_type}): {e}", "ERROR")
+
+            # Exponential backoff between retries
+            if attempt < self.max_retries:
+                backoff = min(2 ** (attempt - 1), 5)  # Max 5 seconds
+                self._log(f"Retrying in {backoff}s...", "DEBUG")
+                await asyncio.sleep(backoff)
+
+        # All retries exhausted
+        self._log(f"All retries exhausted for {domain} ({record_type}). Last error: {last_exception}", "ERROR")
+        return []
+
+    async def resolve(
+            self,
+            url: str,
+            record_type: str = 'A'
+    ) -> List[str]:
+        """
+        Resolve domain with automatic retry and resource management.
+
+        Args:
+            url: URL or domain to resolve
+            record_type: DNS record type (A, AAAA, CNAME, MX, TXT, NS, SOA, PTR)
+
+        Returns:
+            List of resolved records
+        """
+        domain = self._extract_domain(url)
+
+        if not domain:
+            self._log(f"Invalid domain: {url}", "ERROR")
+            return []
+
+        try:
+            # Get shared resolver instance
+            resolver = await self._get_resolver()
+            return await self._query_with_retry(domain, record_type, resolver)
+        except (KeyboardInterrupt, asyncio.CancelledError):
+            return []
+        except Exception as e:
+            self._log(f"Fatal error resolving {domain}: {e}", "ERROR")
+            return []
+
+    async def resolve_many(
+            self,
+            domains: List[str],
+            record_type: str = 'A'
+    ) -> dict[str, List[str]]:
+        """
+        Resolve multiple domains concurrently.
+
+        Note: Concurrency control should be handled by the caller if needed.
+        This method will execute all queries concurrently without limits.
+
+        Args:
+            domains: List of URLs/domains to resolve
+            record_type: DNS record type
+
+        Returns:
+            Dictionary mapping domain to resolved records
+        """
+        tasks = [self.resolve(domain, record_type) for domain in domains]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        return {
+            domain: result if isinstance(result, list) else []
+            for domain, result in zip(domains, results)
+        }
+
+    async def close(self):
+        """
+        Clean up DNS resolver resources.
+
+        Should be called when done with the client to properly release resources.
+        """
+        self._log("Closing DNS client", "INFO")
+        self._resolver = None
+
+    async def resolve_with_fallback(
+            self,
+            url: str,
+            record_types: List[str] = None
+    ) -> dict[str, List[str]]:
+        """
+        Resolve domain with multiple record types as fallback.
+
+        Args:
+            url: URL or domain to resolve
+            record_types: List of record types to try (default: A, AAAA)
+
+        Returns:
+            Dictionary mapping record type to resolved records
+        """
+        if record_types is None:
+            record_types = ['A', 'AAAA']
+
+        results = {}
+        for record_type in record_types:
+            records = await self.resolve(url, record_type)
+            if records:
+                results[record_type] = records
+
+        return results
+
+
+async def main():
+    # Initialize DNS client
+    dns_client = AsyncDnsClient(
+        nameservers=["1.1.1.1", "8.8.8.8"],
+        max_retries=3,
+        timeout=5.0,
+        verbose=True
+    )
+
+    # Single resolution
+    print("=== Single Resolution ===")
+    results = await dns_client.resolve("example.com", "A")
+    print(f"A records for example.com: {results}\n")
+
+    # Multiple resolutions
+    print("=== Multiple Resolutions ===")
+    domains = ["google.com", "github.com", "cloudflare.com"]
+    results = await dns_client.resolve_many(domains, "A")
+    for domain, ips in results.items():
+        print(f"{domain}: {ips}")
+
+    # Resolution with fallback
+    print("\n=== Resolution with Fallback ===")
+    fallback_results = await dns_client.resolve_with_fallback("google.com", ["A", "AAAA"])
+    for record_type, records in fallback_results.items():
+        print(f"{record_type}: {records}")
+
+    # Different record types
+    print("\n=== Different Record Types ===")
+    mx_records = await dns_client.resolve("gmail.com", "MX")
+    print(f"MX records for gmail.com: {mx_records}")
+
+    ns_records = await dns_client.resolve("google.com", "NS")
+    print(f"NS records for google.com: {ns_records}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

subprober/hash/hash.py

@@ -0,0 +1,52 @@
+import hashlib
+import mmh3
+from simhash import Simhash
+from revoltlogger import Logger
+
+def tokenize_for_simhash(text: str) -> list[str]:
+    return text.lower().split()
+
+class HashGen:
+    def __init__(self, algorithms: list[str], verbose: bool = False):
+        self.algorithms = [alg.strip().lower() for alg in algorithms]
+        self.verbose = verbose
+        self.logger = Logger()
+
+    async def gen(self, response: str) -> dict[str, str]:
+        hashed_results = {}
+        processed_response = ""
+
+        try:
+            processed_response = response.encode("utf-8", errors="ignore").decode("utf-8", errors="ignore")
+        except Exception as e:
+            if self.verbose:
+                self.logger.warn(f"Error processing response encoding for hashing: {e}, {type(e)}")
+            return {}
+
+        for alg in self.algorithms:
+            try:
+                if alg == "md5":
+                    hashed_results[alg] = hashlib.md5(processed_response.encode("utf-8")).hexdigest()
+                elif alg == "sha1":
+                    hashed_results[alg] = hashlib.sha1(processed_response.encode("utf-8")).hexdigest()
+                elif alg == "sha256":
+                    hashed_results[alg] = hashlib.sha256(processed_response.encode("utf-8")).hexdigest()
+                elif alg == "sha512":
+                    hashed_results[alg] = hashlib.sha512(processed_response.encode("utf-8")).hexdigest()
+                elif alg == "mmh3":
+                    hashed_results[alg] = str(mmh3.hash(processed_response.encode("utf-8")))
+                elif alg == "simhash":
+                    tokens = tokenize_for_simhash(processed_response)
+                    if tokens:
+                        hashed_results[alg] = str(Simhash(tokens).value)
+                    else:
+                        hashed_results[alg] = ""
+                        if self.verbose:
+                            self.logger.warn("Simhash received empty tokens for response. Setting empty hash.")
+                else:
+                    if self.verbose:
+                        self.logger.warn(f"Undefined or unsupported hash algorithm requested: '{alg}'")
+            except Exception as inner_e:
+                self.logger.warn(f"Error generating hash for algorithm '{alg}': {inner_e}, {type(inner_e)}")
+
+        return hashed_results