bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/modules/virustotal.py

@@ -15,37 +15,15 @@ class virustotal(subdomain_enum_apikey):
     options_desc = {"api_key": "VirusTotal API Key"}
 
     base_url = "https://www.virustotal.com/api/v3"
+    api_page_iter_kwargs = {"json": False, "next_key": lambda r: r.json().get("links", {}).get("next", "")}
 
-    async def setup(self):
-        self.api_key = self.config.get("api_key", "")
-        self.headers = {"x-apikey": self.api_key}
-        return await super().setup()
+    def make_url(self, query):
+        return f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains"
 
-    async def ping(self):
-        # virustotal does not have a ping function
-        return
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["x-apikey"] = self.api_key
+        return url, kwargs
 
-    def parse_results(self, r, query):
-        results = set()
+    async def parse_results(self, r, query):
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
-
-    async def query(self, query):
-        results = set()
-        url = f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains"
-        agen = self.helpers.api_page_iter(
-            url, json=False, headers=self.headers, next_key=lambda r: r.json().get("links", {}).get("next", "")
-        )
-        try:
-            async for response in agen:
-                r = self.parse_results(response, query)
-                if not r:
-                    break
-                results.update(r)
-        finally:
-            agen.aclose()
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)

bbot/modules/wafw00f.py

@@ -52,7 +52,7 @@ class wafw00f(BaseModule):
                     context=f"{{module}} scanned {url} and identified {{event.type}}: {waf}",
                 )
         else:
-            if self.config.get("generic_detect") == True:
+            if self.config.get("generic_detect") is True:
                 generic = await self.helpers.run_in_executor(WW.genericdetect)
                 if generic:
                     waf = "generic detection"

bbot/modules/wayback.py

@@ -10,7 +10,7 @@ class wayback(subdomain_enum):
     meta = {
         "description": "Query archive.org's API for subdomains",
         "created_date": "2022-04-01",
-        "author": "@pmueller",
+        "author": "@liquidsec",
     }
     options = {"urls": False, "garbage_threshold": 10}
     options_desc = {

bbot/modules/wpscan.py

@@ -14,26 +14,26 @@ class wpscan(BaseModule):
 
     options = {
         "api_key": "",
-        "enumerate": "vp,vt,tt,cb,dbe,u,m",
+        "enumerate": "vp,vt,cb,dbe",
         "threads": 5,
-        "request_timeout": 60,
-        "connection_timeout": 30,
+        "request_timeout": 5,
+        "connection_timeout": 2,
         "disable_tls_checks": True,
         "force": False,
     }
     options_desc = {
         "api_key": "WPScan API Key",
-        "enumerate": "Enumeration Process see wpscan help documentation (default: vp,vt,tt,cb,dbe,u,m)",
+        "enumerate": "Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)",
         "threads": "How many wpscan threads to spawn (default is 5)",
-        "request_timeout": "The request timeout in seconds (default 60)",
-        "connection_timeout": "The connection timeout in seconds (default 30)",
+        "request_timeout": "The request timeout in seconds (default 5)",
+        "connection_timeout": "The connection timeout in seconds (default 2)",
         "disable_tls_checks": "Disables the SSL/TLS certificate verification (Default True)",
         "force": "Do not check if the target is running WordPress or returns a 403",
     }
     deps_apt = ["curl", "make", "gcc"]
     deps_ansible = [
         {
-            "name": "Install Ruby Deps (Debian/Ubuntu)",
+            "name": "Install Ruby Deps (Debian)",
             "package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"},
             "become": True,
             "when": "ansible_facts['os_family'] == 'Debian'",
@@ -48,7 +48,13 @@ class wpscan(BaseModule):
             "name": "Install Ruby Deps (Fedora)",
             "package": {"name": ["rubygems", "ruby-devel"], "state": "present"},
             "become": True,
-            "when": "ansible_facts['os_family'] == 'Fedora'",
+            "when": "ansible_facts['os_family'] == 'RedHat'",
+        },
+        {
+            "name": "Install Ruby Deps (Alpine)",
+            "package": {"name": ["ruby-dev", "ruby-bundler"], "state": "present"},
+            "become": True,
+            "when": "ansible_facts['os_family'] == 'Alpine'",
         },
         {
             "name": "Install wpscan gem",
@@ -61,11 +67,11 @@ class wpscan(BaseModule):
         self.processed = set()
         self.ignore_events = ["xmlrpc", "readme"]
         self.api_key = self.config.get("api_key", "")
-        self.enumerate = self.config.get("enumerate", "vp,vt,tt,cb,dbe,u,m")
+        self.enumerate = self.config.get("enumerate", "vp,vt,cb,dbe")
         self.proxy = self.scan.web_config.get("http_proxy", "")
         self.threads = self.config.get("threads", 5)
-        self.request_timeout = self.config.get("request_timeout", 60)
-        self.connection_timeout = self.config.get("connection_timeout", 30)
+        self.request_timeout = self.config.get("request_timeout", 5)
+        self.connection_timeout = self.config.get("connection_timeout", 2)
         self.disable_tls_checks = self.config.get("disable_tls_checks", True)
         self.force = self.config.get("force", False)
         return True

bbot/modules/zoomeye.py

@@ -22,13 +22,16 @@ class zoomeye(subdomain_enum_apikey):
 
     async def setup(self):
         self.max_pages = self.config.get("max_pages", 20)
-        self.headers = {"API-KEY": self.config.get("api_key", "")}
         self.include_related = self.config.get("include_related", False)
         return await super().setup()
 
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["API-KEY"] = self.api_key
+        return url, kwargs
+
     async def ping(self):
         url = f"{self.base_url}/resources-info"
-        r = await self.helpers.request(url, headers=self.headers)
+        r = await self.api_request(url)
         assert int(r.json()["quota_info"]["remain_total_quota"]) > 0, "No quota remaining"
 
     async def handle_event(self, event):
@@ -54,10 +57,10 @@ class zoomeye(subdomain_enum_apikey):
         query_type = 0 if self.include_related else 1
         url = f"{self.base_url}/domain/search?q={self.helpers.quote(query)}&type={query_type}&page=" + "{page}"
         i = 0
-        agen = self.helpers.api_page_iter(url, headers=self.headers)
+        agen = self.api_page_iter(url)
         try:
             async for j in agen:
-                r = list(self.parse_results(j))
+                r = list(await self.parse_results(j))
                 if r:
                     results.update(set(r))
                 if not r or i >= (self.max_pages - 1):
@@ -67,6 +70,8 @@ class zoomeye(subdomain_enum_apikey):
             agen.aclose()
         return results
 
-    def parse_results(self, r):
+    async def parse_results(self, r):
+        results = set()
         for entry in r.get("list", []):
-            yield entry["name"]
+            results.add(entry["name"])
+        return results

bbot/presets/baddns-thorough.yml

@@ -0,0 +1,12 @@
+description: Run all baddns modules and submodules.
+
+
+modules:
+  - baddns
+  - baddns_zone
+  - baddns_direct
+
+config:
+  modules:
+    baddns:
+      enabled_submodules: [CNAME,references,MX,NS,TXT]

bbot/presets/fast.yml

@@ -0,0 +1,16 @@
+description: Scan only the provided targets as fast as possible - no extra discovery
+
+exclude_modules:
+  - excavate
+
+config:
+  # only scan the exact targets specified
+  scope:
+    strict: true
+  # speed up dns resolution by doing A/AAAA only - not MX/NS/SRV/etc
+  dns:
+    minimal: true
+  # essential speculation only
+  modules:
+    speculate:
+      essential_only: true

bbot/presets/kitchen-sink.yml

@@ -10,10 +10,9 @@ include:
   - paramminer
   - dirbust-light
   - web-screenshots
+  - baddns-thorough
 
 config:
   modules:
     baddns:
       enable_references: True
-
-

bbot/presets/spider.yml

@@ -3,6 +3,10 @@ description: Recursive web spider
 modules:
   - httpx
 
+blacklist:
+  # Prevent spider from invalidating sessions by logging out
+  - "RE:/.*(sign|log)[_-]?out"
+
 config:
   web:
     # how many links to follow in a row

bbot/presets/subdomain-enum.yml

@@ -13,10 +13,10 @@ config:
     threads: 25
     brute_threads: 1000
   # put your API keys here
-  modules:
-    github:
-      api_key: ""
-    chaos:
-      api_key: ""
-    securitytrails:
-      api_key: ""
+  # modules:
+  #   github:
+  #     api_key: ""
+  #   chaos:
+  #     api_key: ""
+  #   securitytrails:
+  #     api_key: ""

bbot/presets/web/dotnet-audit.yml

@@ -19,4 +19,3 @@ config:
       extensions: asp,aspx,ashx,asmx,ascx
     telerik:
       exploit_RAU_crypto: True
-

bbot/scanner/manager.py

@@ -1,10 +1,10 @@
 import asyncio
 from contextlib import suppress
 
-from bbot.modules.base import InterceptModule
+from bbot.modules.base import BaseInterceptModule
 
 
-class ScanIngress(InterceptModule):
+class ScanIngress(BaseInterceptModule):
     """
     This is always the first intercept module in the chain, responsible for basic scope checks
 
@@ -15,9 +15,7 @@ class ScanIngress(InterceptModule):
     # accept all events regardless of scope distance
     scope_distance_modifier = None
     _name = "_scan_ingress"
-
-    # small queue size so we don't drain modules' outgoing queues
-    _qsize = 10
+    _qsize = -1
 
     @property
     def priority(self):
@@ -40,7 +38,7 @@ class ScanIngress(InterceptModule):
             - It also marks the Scan object as finished with initialization by setting `_finished_init` to True.
         """
         if events is None:
-            events = self.scan.target.events
+            events = self.scan.target.seeds.events
         async with self.scan._acatch(self.init_events), self._task_counter.count(self.init_events):
             sorted_events = sorted(events, key=lambda e: len(e.data))
             for event in [self.scan.root_event] + sorted_events:
@@ -51,7 +49,6 @@ class ScanIngress(InterceptModule):
                     event.parent = self.scan.root_event
                 if event.module is None:
                     event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET")
-                event.add_tag("target")
                 if event != self.scan.root_event:
                     event.discovery_context = f"Scan {self.scan.name} seeded with " + "{event.type}: {event.data}"
                 self.verbose(f"Target: {event}")
@@ -115,14 +112,6 @@ class ScanIngress(InterceptModule):
         # nerf event's priority if it's not in scope
         event.module_priority += event.scope_distance
 
-    async def forward_event(self, event, kwargs):
-        # if a module qualifies for "quick-emit", we skip all the intermediate modules like dns and cloud
-        # and forward it straight to the egress module
-        if event.quick_emit:
-            await self.scan.egress_module.queue_event(event, kwargs)
-        else:
-            await super().forward_event(event, kwargs)
-
     @property
     def non_intercept_modules(self):
         if self._non_intercept_modules is None:
@@ -169,7 +158,7 @@ class ScanIngress(InterceptModule):
         return False
 
 
-class ScanEgress(InterceptModule):
+class ScanEgress(BaseInterceptModule):
     """
     This is always the last intercept module in the chain, responsible for executing and acting on the
     `abort_if` and `on_success_callback` functions.

bbot/scanner/preset/args.py

@@ -10,7 +10,6 @@ log = logging.getLogger("bbot.presets.args")
 
 
 class BBOTArgs:
-
     # module config options to exclude from validation
     exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|module_threads)$")
 
@@ -91,18 +90,9 @@ class BBOTArgs:
             *self.parsed.targets,
             whitelist=self.parsed.whitelist,
             blacklist=self.parsed.blacklist,
-            strict_scope=self.parsed.strict_scope,
             name="args_preset",
         )
 
-        # then we set verbosity levels (so if the user enables -d they can see debug output)
-        if self.parsed.silent:
-            args_preset.silent = True
-        if self.parsed.verbose:
-            args_preset.verbose = True
-        if self.parsed.debug:
-            args_preset.debug = True
-
         # then we load requested preset
         # this is important so we can load custom module directories, pull in custom flags, module config options, etc.
         for preset_arg in self.parsed.preset:
@@ -113,6 +103,14 @@ class BBOTArgs:
             except Exception as e:
                 raise BBOTArgumentError(f'Error parsing preset "{preset_arg}": {e}')
 
+        # then we set verbosity levels (so if the user enables -d they can see debug output)
+        if self.parsed.silent:
+            args_preset.silent = True
+        if self.parsed.verbose:
+            args_preset.verbose = True
+        if self.parsed.debug:
+            args_preset.debug = True
+
         # modules + flags
         args_preset.exclude_modules.update(set(self.parsed.exclude_modules))
         args_preset.exclude_flags.update(set(self.parsed.exclude_flags))
@@ -142,9 +140,15 @@ class BBOTArgs:
             args_preset.core.custom_config["deps_behavior"] = "ignore_failed"
 
         # other scan options
-        args_preset.scan_name = self.parsed.name
-        args_preset.output_dir = self.parsed.output_dir
-        args_preset.force_start = self.parsed.force
+        if self.parsed.name is not None:
+            args_preset.scan_name = self.parsed.name
+        if self.parsed.output_dir is not None:
+            args_preset.output_dir = self.parsed.output_dir
+        if self.parsed.force:
+            args_preset.force_start = self.parsed.force
+
+        if self.parsed.proxy:
+            args_preset.core.merge_custom({"web": {"http_proxy": self.parsed.proxy}})
 
         if self.parsed.custom_headers:
             args_preset.core.merge_custom({"web": {"http_headers": self.parsed.custom_headers}})
@@ -162,13 +166,19 @@ class BBOTArgs:
             except Exception as e:
                 raise BBOTArgumentError(f'Error parsing command-line config option: "{config_arg}": {e}')
 
+        # strict scope
+        if self.parsed.strict_scope:
+            args_preset.core.merge_custom({"scope": {"strict": True}})
+
         return args_preset
 
     def create_parser(self, *args, **kwargs):
         kwargs.update(
-            dict(
-                description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=self.epilog
-            )
+            {
+                "description": "Bighuge BLS OSINT Tool",
+                "formatter_class": argparse.RawTextHelpFormatter,
+                "epilog": self.epilog,
+            }
         )
         p = argparse.ArgumentParser(*args, **kwargs)
 
@@ -206,7 +216,7 @@ class BBOTArgs:
             metavar="CONFIG",
             default=[],
         )
-        presets.add_argument("-lp", "--list-presets", action="store_true", help=f"List available presets.")
+        presets.add_argument("-lp", "--list-presets", action="store_true", help="List available presets.")
 
         modules = p.add_argument_group(title="Modules")
         modules.add_argument(
@@ -214,31 +224,31 @@ class BBOTArgs:
             "--modules",
             nargs="+",
             default=[],
-            help=f'Modules to enable. Choices: {",".join(self.preset.module_loader.scan_module_choices)}',
+            help=f'Modules to enable. Choices: {",".join(sorted(self.preset.module_loader.scan_module_choices))}',
             metavar="MODULE",
         )
-        modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.")
+        modules.add_argument("-l", "--list-modules", action="store_true", help="List available modules.")
         modules.add_argument(
             "-lmo", "--list-module-options", action="store_true", help="Show all module config options"
         )
         modules.add_argument(
-            "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE"
+            "-em", "--exclude-modules", nargs="+", default=[], help="Exclude these modules.", metavar="MODULE"
         )
         modules.add_argument(
             "-f",
             "--flags",
             nargs="+",
             default=[],
-            help=f'Enable modules by flag. Choices: {",".join(self.preset.module_loader.flag_choices)}',
+            help=f'Enable modules by flag. Choices: {",".join(sorted(self.preset.module_loader.flag_choices))}',
             metavar="FLAG",
         )
-        modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.")
+        modules.add_argument("-lf", "--list-flags", action="store_true", help="List available flags.")
         modules.add_argument(
             "-rf",
             "--require-flags",
             nargs="+",
             default=[],
-            help=f"Only enable modules with these flags (e.g. -rf passive)",
+            help="Only enable modules with these flags (e.g. -rf passive)",
             metavar="FLAG",
         )
         modules.add_argument(
@@ -246,7 +256,7 @@ class BBOTArgs:
             "--exclude-flags",
             nargs="+",
             default=[],
-            help=f"Disable modules with these flags. (e.g. -ef aggressive)",
+            help="Disable modules with these flags. (e.g. -ef aggressive)",
             metavar="FLAG",
         )
         modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules")
@@ -262,7 +272,12 @@ class BBOTArgs:
             help="Run scan even in the case of condition violations or failed module setups",
         )
         scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt")
-        scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan")
+        scan.add_argument(
+            "--fast-mode",
+            action="store_true",
+            help="Scan only the provided targets as fast as possible, with no extra discovery",
+        )
+        scan.add_argument("--dry-run", action="store_true", help="Abort before executing scan")
         scan.add_argument(
             "--current-preset",
             action="store_true",
@@ -286,7 +301,7 @@ class BBOTArgs:
             "--output-modules",
             nargs="+",
             default=[],
-            help=f'Output module(s). Choices: {",".join(self.preset.module_loader.output_module_choices)}',
+            help=f'Output module(s). Choices: {",".join(sorted(self.preset.module_loader.output_module_choices))}',
             metavar="MODULE",
         )
         output.add_argument("--json", "-j", action="store_true", help="Output scan data in JSON format")
@@ -307,6 +322,7 @@ class BBOTArgs:
 
         misc = p.add_argument_group(title="Misc")
         misc.add_argument("--version", action="store_true", help="show BBOT version and exit")
+        misc.add_argument("--proxy", help="Use this proxy for all HTTP requests", metavar="HTTP_PROXY")
         misc.add_argument(
             "-H",
             "--custom-headers",
@@ -356,6 +372,10 @@ class BBOTArgs:
             custom_headers_dict[k] = v
         self.parsed.custom_headers = custom_headers_dict
 
+        # --fast-mode
+        if self.parsed.fast_mode:
+            self.parsed.preset += ["fast"]
+
     def validate(self):
         # validate config options
         sentinel = object()

bbot/scanner/preset/environ.py

@@ -6,6 +6,9 @@ from pathlib import Path
 from bbot.core.helpers.misc import cpu_architecture, os_platform, os_platform_friendly
 
 
+REQUESTS_PATCHED = False
+
+
 def increase_limit(new_limit):
     try:
         import resource
@@ -62,7 +65,6 @@ omegaconf.OmegaConf.register_new_resolver("env", env_resolver)
 
 
 class BBOTEnviron:
-
     def __init__(self, preset):
         self.preset = preset
 
@@ -120,7 +122,10 @@ class BBOTEnviron:
 
         urllib3.disable_warnings()
         ssl_verify = self.preset.config.get("ssl_verify", False)
-        if not ssl_verify:
+
+        global REQUESTS_PATCHED
+        if not ssl_verify and not REQUESTS_PATCHED:
+            REQUESTS_PATCHED = True
             import requests
             import functools
 

bbot/scanner/preset/path.py

@@ -33,13 +33,16 @@ class PresetPath:
         if "/" in str(filename):
             if filename_path.parent not in paths_to_search:
                 paths_to_search.append(filename_path.parent)
-        log.debug(f"Searching for preset in {paths_to_search}, file candidates: {file_candidates_str}")
+        log.debug(
+            f"Searching for preset in {[str(p) for p in paths_to_search]}, file candidates: {file_candidates_str}"
+        )
         for path in paths_to_search:
             for candidate in file_candidates:
                 for file in path.rglob(candidate):
-                    log.verbose(f'Found preset matching "{filename}" at {file}')
-                    self.add_path(file.parent)
-                    return file.resolve()
+                    if file.is_file():
+                        log.verbose(f'Found preset matching "{filename}" at {file}')
+                        self.add_path(file.parent)
+                        return file.resolve()
         raise ValidationError(
             f'Could not find preset at "{filename}" - file does not exist. Use -lp to list available presets'
         )