ansferatu 0.1.0__py3-none-any.whl

ansferatu/__init__.py

@@ -0,0 +1,19 @@
+"""Ansferatu - multifunctional tool for HTTP reconnaissance, web crawling and directory bruteforce.
+
+Public library API
+------------------
+The most common entry points for using Ansferatu programmatically are the
+high-level orchestration helpers::
+
+    from ansferatu import common_crawler, common_brute_from_file
+
+For lower-level control you can build a spider directly::
+
+    from ansferatu.spider import WebSpider, TaskFetch
+"""
+
+__version__ = "0.1.0"
+
+from ansferatu.profiles.modes import common_crawler, common_brute_from_file
+
+__all__ = ["common_crawler", "common_brute_from_file", "__version__"]

ansferatu/__main__.py

@@ -0,0 +1,4 @@
+from ansferatu.cli import main
+
+if __name__ == "__main__":
+    main()

ansferatu/cli.py

@@ -0,0 +1,252 @@
+import argparse
+
+from datetime import datetime
+import logging
+import os
+from ansferatu.spider.common.url import Url
+from ansferatu.profiles.modes import common_crawler
+from ansferatu.profiles.modes import common_brute_from_file
+
+# Create class instance and timestamp for report file
+now = datetime.now()
+
+# configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s\t%(levelname)s\t%(message)s")
+logger = logging.getLogger()
+
+class AnsferatuRunner:
+    def __init__(self, args):
+        logger.info("Welcome to Ansferatu")
+        self.timestamp_start = datetime.timestamp(now)
+
+        self.runner_args = args
+
+        if self.runner_args.input_file:
+            self.url_list = open(self.runner_args.input_file, "r+").read().splitlines()
+        else:
+            self.url_list = self.runner_args.url_list_initial
+
+        if not self.runner_args.scope_list:
+            self.new_scope = self.create_scope()
+        else:
+            self.new_scope = self.runner_args.scope_list
+
+        if not os.path.exists(self.runner_args.output_dir):
+            os.makedirs(self.runner_args.output_dir)
+
+    def create_scope(self):
+        scope_domains_list = []
+        for url in self.url_list:
+            n_url = Url(url)
+            scope_domains_list.append(str(n_url.domain))
+
+        return scope_domains_list
+
+    def run(self):
+        if self.runner_args.mode == "crawl":
+            self.crawl()
+        elif self.runner_args.mode == "brute":
+            self.brute()
+        self.dump_result()
+
+    def crawl(self):
+        common_crawler(
+            self.url_list,
+            self.new_scope,
+            max_deep=self.runner_args.max_deep,
+            exclude_codes_list=self.runner_args.exclude_codes_list,
+            visit_count_limit=self.runner_args.visit_count_limit,
+            threads=self.runner_args.threads,
+            headless=getattr(self.runner_args, 'headless', False),
+            fill_forms=getattr(self.runner_args, 'fill_forms', False),
+            output_file=self.runner_args.jsonl_output,
+            skip_dedup=self.runner_args.skip_dedup,
+            headless_workers=getattr(self.runner_args, 'headless_workers', 1),
+            max_headless_mem_mb=getattr(self.runner_args, 'max_headless_mem_mb', None),
+        )
+
+    def brute(self):
+        common_brute_from_file(
+            self.url_list,
+            self.runner_args.exclude_codes_list,
+            _dictionary_file=self.runner_args.wordlist_big,
+            threads=self.runner_args.threads,
+            output_file=self.runner_args.jsonl_output,
+            skip_dedup=self.runner_args.skip_dedup,
+        )
+
+    def dump_result(self):
+        # Dump results in file
+        for domain in self.new_scope:
+            logger.info("creating output file...")
+            result_file = f"{self.runner_args.output_dir}final-{domain}+{self.timestamp_start}.{self.runner_args.report_mode}"
+
+
+def parse_arguments(argv=None):
+    parser = argparse.ArgumentParser()
+
+    # Usual arguments which are applicable for the whole script / top-level args
+    parser.add_argument("--verbose", help="Common top-level parameter", action="store_true", required=False)
+
+    # Same subparsers as usual
+    subparsers = parser.add_subparsers(help="Desired action to perform", dest="mode")
+
+    # Usual subparsers not using common options
+    # parser_other = subparsers.add_parser("extra-action", help='Do something without db')
+
+    # Create parent subparser. Note `add_help=False` and creation via `argparse.`
+    parent_parser = argparse.ArgumentParser(add_help=False)
+    parent_parser.add_argument(
+        "-o",
+        "--output",
+        metavar="Output Directory",
+        type=str,
+        dest="output_dir",
+        default="/tmp/",
+        help="Output directory for reports. By default: /tmp/",
+    )
+    parent_parser.add_argument(
+        "--jsonl",
+        metavar="FILE",
+        type=str,
+        dest="jsonl_output",
+        default=None,
+        help="Path to JSONL output file (Nuclei-compatible proxify format)",
+    )
+    parent_parser.add_argument(
+        "-s",
+        "--scope",
+        metavar="InScope domains",
+        dest="scope_list",
+        nargs="+",
+        default=[],
+        help="List of domains that are in scope of current scan. Split by space. "
+        "Example: news.example.com blog.example.com admin.example.com",
+    )
+    parent_parser.add_argument(
+        "--report",
+        choices=["html", "json"],
+        default="html",
+        dest="report_mode",
+        help="Type of report format",
+    )
+    parent_parser.add_argument(
+        "--exclude_codes",
+        default=[403, 404, 401],
+        nargs="*",
+        dest="exclude_codes_list",
+        help="List of filtered HTTP answer's codes",
+    )
+    parent_parser.add_argument(
+        "--threads",
+        "-t",
+        metavar="Max_threads",
+        type=int,
+        dest="threads",
+        default=10,
+        help="Number of HTTP threads",
+    )
+    parent_parser.add_argument(
+        "--sd",
+        "--skip-deduplication",
+        action="store_true",
+        default=False,
+        dest="skip_dedup",
+        help="Disable response deduplication (keep all responses)",
+    )
+
+    group = parent_parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        "-u",
+        "--urls",
+        metavar="URLs",
+        type=str,
+        dest="url_list_initial",
+        nargs="*",
+        help="URLs or list of URLs for start. Example: https://mail.ru/",
+    )
+    group.add_argument(
+        "-f",
+        "--file",
+        metavar="Input file with URLs",
+        type=str,
+        dest="input_file",
+        help="Input file contains URLs (1 URL per line)",
+    )
+
+    parser_crawl = subparsers.add_parser("crawl", parents=[parent_parser], help="Create something")
+    parser_crawl.add_argument(
+        "--limit",
+        metavar="Limits",
+        type=int,
+        dest="visit_count_limit",
+        default=10,
+        help="Limit of request to subdirectories",
+    )
+    parser_crawl.add_argument(
+        "--deep",
+        metavar="Max_deep",
+        type=int,
+        dest="max_deep",
+        default=2,
+        help="Max deep of request to subdirectories",
+    )
+    parser_crawl.add_argument(
+        "--headless",
+        action="store_true",
+        default=False,
+        help="Enable Playwright headless extraction for root/absolute URLs",
+    )
+    parser_crawl.add_argument(
+        "--fill-forms",
+        action="store_true",
+        default=False,
+        dest="fill_forms",
+        help="Enable form detection and interaction (requires --headless). "
+             "Detects forms on pages, fills fields with smart defaults, and submits them.",
+    )
+    parser_crawl.add_argument(
+        "--headless-workers",
+        metavar="N",
+        type=int,
+        dest="headless_workers",
+        default=1,
+        help="Number of parallel headless workers (HeadlessExtractor + FormInteractor). "
+             "Each worker spawns its own Chromium (~250MB RAM, ~1 CPU under load). "
+             "Default 1 preserves legacy behaviour; raise to 3-5 to drain a backlog faster.",
+    )
+    parser_crawl.add_argument(
+        "--max-headless-mem-mb",
+        metavar="MB",
+        type=int,
+        dest="max_headless_mem_mb",
+        default=None,
+        help="Hard cap on estimated headless RAM usage (MB).  When set and the "
+             "estimate (workers * 2 browsers * ~350MB) exceeds it, startup aborts. "
+             "When unset, an advisory warning is logged if estimate exceeds 70%% "
+             "of currently available RAM.",
+    )
+
+    parser_brute = subparsers.add_parser("brute", parents=[parent_parser], help="Update something")
+    parser_brute.add_argument(
+        "-w",
+        "--wordlist",
+        required=True,
+        metavar="=wordlist file",
+        type=str,
+        dest="wordlist_big",
+        help="Path to wordlist file for brute",
+    )
+
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    """Console entry point for the ``ansferatu`` command."""
+    args = parse_arguments(argv)
+    ansferatu_runner = AnsferatuRunner(args)
+    ansferatu_runner.run()
+
+
+if __name__ == "__main__":
+    main()

ansferatu/profiles/CommonExtractor.py

@@ -0,0 +1,309 @@
+from ansferatu import spider
+import re
+from ansferatu.spider.common.url import Url
+
+# Cap the size of response bodies we feed into the heavy URL-extraction
+# regex.  Running ``common_regexp_compiled`` (many alternations, VERBOSE
+# mode) over minified JS bundles of 2–10 MB is several seconds of pure
+# Python regex holding the GIL, which starves other threads (notably the
+# Playwright form interactor).  500 KB is enough to catch nearly all real
+# URLs in HTML/JS — anything further is almost always duplicates.
+MAX_EXTRACT_BODY_BYTES = 500 * 1024
+
+# Response types the extractor actually mines for URLs.  Everything
+# else (images, binaries, fonts, large JSON blobs, …) produces no
+# useful output and is skipped to save GIL time.  ``other`` is
+# included because ``detect_response_type`` returns it for responses
+# with no Content-Type header — these are rare but occasionally
+# contain useful URLs (e.g. raw redirect bodies).
+_EXTRACTABLE_TYPES = frozenset({"html", "js", "robots", "sitemap", "other"})
+
+
+class CommonExtractor(spider.Extractor):
+    """
+    Class for getting new links from response HTMLs, JS files, etc.
+    """
+
+    def __init__(self, max_deep=0, scope_list=None):
+        """
+        """
+        super().__init__()
+        if scope_list is None:
+            scope_list = []
+        self._max_deep = max_deep
+        self._scope_list = scope_list
+
+        self.js_content_types = ["text/javascript", "application/x-ecmascript", "application/x-javascript",
+                                 "application/javascript", "text/ecmascript", "text/x-ecmascript", "text/x-javascript",
+                                 "application/ecmascript", "text/jscript"]
+
+        self.html_content_types = ["text/html"]
+
+        # Regex used in linkfinder
+        self.common_regexp_str = r"""
+
+          (?:"|')                               # Start newline delimiter
+
+          (
+            ((?:[a-zA-Z]{1,10}://|//)           # Match a scheme [a-Z]*1-10 or //
+            [^"'/]{1,}\.                        # Match a domainname (any character + dot)
+            [a-zA-Z]{2,}[^"']{0,})              # The domainextension and/or path
+
+            |
+
+            ((?:/|\.\./|\./)                    # Start with /,../,./
+            [^"'><,;| *()(%%$^/\\\[\]]          # Next character can't be...
+            [^"'><,;|()]{1,})                   # Rest of the characters can't be
+
+            |
+
+            ([a-zA-Z0-9_\-/]{1,}/               # Relative endpoint with /
+            [a-zA-Z0-9_\-/]{1,}                 # Resource name
+            \.(?:[a-zA-Z]{1,4}|action)          # Rest + extension (length 1-4 or action)
+            (?:[\?|/][^"|']{0,}|))              # ? mark with parameters
+
+            |
+
+            ([a-zA-Z0-9_\-]{1,}                 # filename
+            \.(?:|html|php|txt|htm|aspx|
+            asp|js|css|pgsql|mysql|pdf|
+            cgi|inc|gif|jpg|swf|xml|cfm|
+            xhtml|wmv|zip|axd|gz|png|doc|
+            shtml|jsp|ico|exe|csi|config|
+            jpeg|ashx|log|xls|old|mp3|tar
+            |ini|asa|tgz|flv|php3|bak|rar|
+            asmx|xlsx|page|phtml|dll|asax|
+            pl|csv|ppt|bmp|sql|new|avi|psd|
+            rss|wav|action|db|dat|do|xsl|
+            class|mdb|include|cs|htc|mov|
+            mpg|rdf|rtf|ascx|files|jar|vb|
+            mp4|local|docx|php5|wci|readme|
+            cfg|cfc|lck|ttf|jhtml|mpeg|php4|
+            tif|json|zif|shtm|sitemap|tmp|
+            backup|conf|settings|cab|asx|
+            msi|bin|htaccess|java|jsf|bat|
+            print|ics|svc|vbs|img|inf|ajax|
+            chm|m3u|py|sh|store|webinfo|jad|
+            stm|webresource|lock|phps|pptx|
+            xsd|crt|hmtl|index|iso|taf|war|
+            xslt|go|gpx|ihtml|odt|sample|spider|
+            cer|html|lib|lnk|mhtml|pgp|
+            text|view|asc|dtd|html1|ogg|out|
+            pgt|php|rb|rhtml|wsdl|apsx|asf|
+            dot|git|hta|php2|phtm|psql|reg|
+            rpm|tiff|cfml|dta|jp|php~|ps|
+            raw|svg|svn|thtml|xhtm|aspx|
+            bhtml|bml|ca|cache|htmls|htx|
+            jpe|jspf|access|app|asd|asm|bak2|
+            deb|epub|htlm|jnlp|js2|jspx|
+            php1|phpp|pop3|pwd|pyc|session|
+            setup|swp|temp)             		# . + extension
+            (?:\?[^"|']{0,}|))                  # ? mark with parameters
+
+          )
+
+          (?:"|')                               # End newline delimiter
+          
+          |
+            
+            (?<=(?<=href=)|(?<=src=))((?:/|\.\./|\./)[^"'><,;|*()(%%$^/\\\[\]][^"'><,;|()]{1,})
+
+        """
+        self.common_regexp_compiled = re.compile(self.common_regexp_str, re.VERBOSE)
+
+        self.robots_regexp_str = "Allow: (.*)|Disallow: (.*)"
+        self.robots_regexp_compiled = re.compile(self.robots_regexp_str, re.VERBOSE)
+
+        self.sitemap_regexp_str = "<loc>(.*?)</loc>"
+        self.sitemap_regexp_compiled = re.compile(self.sitemap_regexp_str, re.VERBOSE)
+
+        self.regex_common2 = re.compile(r'https?://(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}([-a-zA-Z0-9()@:%_\+.~#?&//=]*)', re.VERBOSE)
+
+        # Regex for finding standalone domain names (not full URLs).
+        # Targets bare domains in quoted strings — JS configs, arrays, HTML attributes, etc.
+        # e.g. "nordvpn.com", 'china-with-nord.org', "sub.domain.example.net"
+        # Uses post-filtering (not_tld_extensions) to discard file-name false positives.
+        self.domain_regexp_str = r"""
+            (?:"|')                                                  # Opening quote
+            (?![a-zA-Z]{1,10}://)                                    # NOT a full URL (http://, ftp://, etc.)
+            (?!//)                                                   # NOT a protocol-relative URL
+            (
+                (?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?  # Domain label (RFC-compliant)
+                \.){1,10}                                            # 1-10 labels, each followed by dot
+                [a-zA-Z]{2,15}                                       # TLD: 2-15 alpha chars
+            )
+            (?:"|')                                                  # Closing quote
+        """
+        self.domain_regexp_compiled = re.compile(self.domain_regexp_str, re.VERBOSE)
+
+        # Whitelist of popular TLDs — only domains ending with these are accepted.
+        # Keeps precision high: no .js/.html/.png false positives possible.
+        self.allowed_tlds = {
+            # Classic gTLDs
+            'com', 'org', 'net', 'edu', 'gov', 'biz', 
+            # New gTLDs (tech / startup / business)
+            'io', 'ai', 'app', 'dev', 'co',
+            'xyz', 'tech', 'site', 'online', 'website', 'cloud',
+            'network', 'group', 'review',
+            'security', 'services', 'solutions', 'support', 'systems',
+            'wiki',
+            # Major country-code TLDs (Europe)
+            'uk', 'de', 'fr', 'nl', 'it', 'es', 'pt', 'eu', 'ch', 'at', 'be',
+            'se', 'no', 'dk', 'fi', 'ie', 'pl', 'cz', 'ro', 'hu', 'bg', 'hr',
+            'sk', 'si', 'lt', 'lv', 'ee', 'gr', 'ua', 'by', 'ru',
+            # Major country-code TLDs (Americas)
+            'us', 'ca', 'mx', 'br', 'ar', 'cl', 'co',
+            # Major country-code TLDs (Asia-Pacific)
+            'cn', 'jp', 'kr', 'in', 'au', 'nz', 'sg', 'hk', 'tw', 'th',
+            'ph', 'id', 'my', 'vn', 'pk',
+            # Major country-code TLDs (Middle East / Africa)
+            'il', 'ae', 'sa', 'tr', 'eg', 'za', 'ng', 'ke',
+        }
+
+    def regexp_search_common(self, data):
+        items = [m.group(1) for m in self.common_regexp_compiled.finditer(data)]
+        items2 = [m.group(0) for m in self.regex_common2.finditer(data)]
+        return items + items2
+
+    def regexp_search_domains(self, data):
+        """Find standalone domain names (not full URLs) in text.
+
+        Targets bare domains in quoted strings — common in JS configs, arrays,
+        GA linker domains, CSP headers, CORS whitelists, etc.
+        Only keeps domains whose TLD is in the allowed_tlds whitelist.
+        Returns deduplicated, lowercased domain list.
+        """
+        raw = [m.group(1) for m in self.domain_regexp_compiled.finditer(data)]
+        filtered = []
+        for domain in raw:
+            tld = domain.rsplit('.', 1)[-1].lower()
+            if tld in self.allowed_tlds:
+                filtered.append(domain.lower())
+        return list(set(filtered))
+
+    def regexp_search_robots(self, data):
+        items = [m.group(2) for m in self.robots_regexp_compiled.finditer(data)]
+        return items
+
+    def regexp_search_sitemap(self, data):
+        items = [m.group(1) for m in self.sitemap_regexp_compiled.finditer(data)]
+        return items
+
+    def detect_response_type(self, response):
+
+        _url = str(response.url)
+
+        if "Content-Type" in response.headers:
+            content_type = response.headers["Content-Type"].split(';')[0]
+
+            if content_type in self.js_content_types or ".js" in _url:
+                return "js"
+
+            elif content_type in self.html_content_types or ".html" in _url:
+                return "html"
+
+            elif "robots.txt" in _url:
+                return "robots"
+
+            elif content_type == "text/xml" and "sitemap" in _url:
+                return "sitemap"
+
+        else:
+            return "other"
+
+    def htm_extract(self, task_extract: spider.TaskExtract) -> spider.ResultExtract:
+        """
+        """
+
+        response = task_extract.content
+        url_now = response.url
+        html_headers = str(response.headers)
+
+        ## STEP 1: Define respose type
+        response_type = self.detect_response_type(response)
+
+        ## STEP 2: Finding new URLs with regexp
+        regexp_headers = self.regexp_search_common(html_headers)
+
+        # Fast path: skip body decode + regex sweep entirely for non-textual
+        # or irrelevant content types.  Also bounds response.text work that
+        # would otherwise run charset-normalizer on binary bodies.
+        if response_type not in _EXTRACTABLE_TYPES:
+            html_text = ""
+            regexp_body = []
+            new_domains2 = []
+        else:
+            # ``response.content`` is already the raw bytes; check size
+            # before decoding so huge JS bundles don't blow the regex path.
+            try:
+                body_size = len(response.content) if response.content is not None else 0
+            except Exception:
+                body_size = 0
+
+            if body_size > MAX_EXTRACT_BODY_BYTES:
+                # Decode only the prefix.  ``response.encoding`` is already
+                # primed to utf-8 by CommonFetcher, so this is a fast C
+                # decode with errors='replace'.
+                encoding = response.encoding or "utf-8"
+                html_text = response.content[:MAX_EXTRACT_BODY_BYTES].decode(
+                    encoding, errors="replace")
+            else:
+                html_text = str(response.text)
+
+            if response_type == "robots":
+                regexp_body = self.regexp_search_robots(html_text)
+            elif response_type == "sitemap":
+                regexp_body = self.regexp_search_sitemap(html_text)
+            elif response_type in ("js", "html"):
+                regexp_body = self.regexp_search_common(html_text)
+            else:  # "other" — no Content-Type; keep original behaviour
+                regexp_body = []
+
+            new_domains2 = self.regexp_search_domains(html_text)
+
+        ## Extract new domains and create new urls:
+
+        new_domains1 = self.regexp_search_domains(html_headers)
+
+        new_domains = new_domains1 + new_domains2
+
+        new_urls = []
+        for domain in new_domains:
+            base = "https://" + domain + "/"
+            new_urls.append(base)
+            new_urls.append(base + "sitemap.xml")
+            new_urls.append(base + "robots.txt")
+
+        if len(new_urls) > 0:
+            pass
+
+        ## STEP 3: Garbage filtration
+
+        url_list_regexp = []
+        for _url in (regexp_body + regexp_headers):
+            if _url is not None:
+                _url_ = spider.get_url_legal(_url, base_url=url_now)
+                url_list_regexp.append(_url_)
+                url_obj = Url(_url_)
+                dirs_list = url_obj.list_all_dirs()
+                url_list_regexp.extend(dirs_list)
+
+
+        ## STEP 4: Add absolute urls, sitemaps and robots to url list
+        additional_url_list = []
+        for url1 in url_list_regexp:
+            url2 = Url(url1)
+            absl_url = url2.absolute
+            additional_url_list.append(absl_url)
+            additional_url_list.append(absl_url + "sitemap.xml")
+            additional_url_list.append(absl_url + "robots.txt")
+
+        ## STEP 5: Combine all lists
+        combined_url_list = url_list_regexp + additional_url_list + new_urls
+
+        final_url_list = {_url for _url in combined_url_list if
+                          spider.check_url_legal_and_in_scope(_url, self._scope_list)}
+        task_fetch_list = [spider.TaskFetch.from_task_extract(task_extract, url_new=url) for url in final_url_list]
+
+        return spider.ResultExtract(state_code=1, task_fetch_list=task_fetch_list)
+