bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/presets/subdomain-enum.yml

@@ -13,10 +13,10 @@ config:
     threads: 25
     brute_threads: 1000
   # put your API keys here
-  modules:
-    github:
-      api_key: ""
-    chaos:
-      api_key: ""
-    securitytrails:
-      api_key: ""
+  # modules:
+  #   github:
+  #     api_key: ""
+  #   chaos:
+  #     api_key: ""
+  #   securitytrails:
+  #     api_key: ""

bbot/scanner/manager.py

@@ -1,10 +1,10 @@
 import asyncio
 from contextlib import suppress
 
-from bbot.modules.base import InterceptModule
+from bbot.modules.base import BaseInterceptModule
 
 
-class ScanIngress(InterceptModule):
+class ScanIngress(BaseInterceptModule):
     """
     This is always the first intercept module in the chain, responsible for basic scope checks
 
@@ -15,9 +15,7 @@ class ScanIngress(InterceptModule):
     # accept all events regardless of scope distance
     scope_distance_modifier = None
     _name = "_scan_ingress"
-
-    # small queue size so we don't drain modules' outgoing queues
-    _qsize = 10
+    _qsize = -1
 
     @property
     def priority(self):
@@ -40,7 +38,7 @@ class ScanIngress(InterceptModule):
             - It also marks the Scan object as finished with initialization by setting `_finished_init` to True.
         """
         if events is None:
-            events = self.scan.target.events
+            events = self.scan.target.seeds.events
         async with self.scan._acatch(self.init_events), self._task_counter.count(self.init_events):
             sorted_events = sorted(events, key=lambda e: len(e.data))
             for event in [self.scan.root_event] + sorted_events:
@@ -51,7 +49,6 @@ class ScanIngress(InterceptModule):
                     event.parent = self.scan.root_event
                 if event.module is None:
                     event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET")
-                event.add_tag("target")
                 if event != self.scan.root_event:
                     event.discovery_context = f"Scan {self.scan.name} seeded with " + "{event.type}: {event.data}"
                 self.verbose(f"Target: {event}")
@@ -115,14 +112,6 @@ class ScanIngress(InterceptModule):
         # nerf event's priority if it's not in scope
         event.module_priority += event.scope_distance
 
-    async def forward_event(self, event, kwargs):
-        # if a module qualifies for "quick-emit", we skip all the intermediate modules like dns and cloud
-        # and forward it straight to the egress module
-        if event.quick_emit:
-            await self.scan.egress_module.queue_event(event, kwargs)
-        else:
-            await super().forward_event(event, kwargs)
-
     @property
     def non_intercept_modules(self):
         if self._non_intercept_modules is None:
@@ -169,7 +158,7 @@ class ScanIngress(InterceptModule):
         return False
 
 
-class ScanEgress(InterceptModule):
+class ScanEgress(BaseInterceptModule):
     """
     This is always the last intercept module in the chain, responsible for executing and acting on the
     `abort_if` and `on_success_callback` functions.

bbot/scanner/preset/args.py

@@ -10,7 +10,6 @@ log = logging.getLogger("bbot.presets.args")
 
 
 class BBOTArgs:
-
     # module config options to exclude from validation
     exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|module_threads)$")
 
@@ -91,18 +90,9 @@ class BBOTArgs:
             *self.parsed.targets,
             whitelist=self.parsed.whitelist,
             blacklist=self.parsed.blacklist,
-            strict_scope=self.parsed.strict_scope,
             name="args_preset",
         )
 
-        # then we set verbosity levels (so if the user enables -d they can see debug output)
-        if self.parsed.silent:
-            args_preset.silent = True
-        if self.parsed.verbose:
-            args_preset.verbose = True
-        if self.parsed.debug:
-            args_preset.debug = True
-
         # then we load requested preset
         # this is important so we can load custom module directories, pull in custom flags, module config options, etc.
         for preset_arg in self.parsed.preset:
@@ -113,6 +103,14 @@ class BBOTArgs:
             except Exception as e:
                 raise BBOTArgumentError(f'Error parsing preset "{preset_arg}": {e}')
 
+        # then we set verbosity levels (so if the user enables -d they can see debug output)
+        if self.parsed.silent:
+            args_preset.silent = True
+        if self.parsed.verbose:
+            args_preset.verbose = True
+        if self.parsed.debug:
+            args_preset.debug = True
+
         # modules + flags
         args_preset.exclude_modules.update(set(self.parsed.exclude_modules))
         args_preset.exclude_flags.update(set(self.parsed.exclude_flags))
@@ -142,9 +140,15 @@ class BBOTArgs:
             args_preset.core.custom_config["deps_behavior"] = "ignore_failed"
 
         # other scan options
-        args_preset.scan_name = self.parsed.name
-        args_preset.output_dir = self.parsed.output_dir
-        args_preset.force_start = self.parsed.force
+        if self.parsed.name is not None:
+            args_preset.scan_name = self.parsed.name
+        if self.parsed.output_dir is not None:
+            args_preset.output_dir = self.parsed.output_dir
+        if self.parsed.force:
+            args_preset.force_start = self.parsed.force
+
+        if self.parsed.proxy:
+            args_preset.core.merge_custom({"web": {"http_proxy": self.parsed.proxy}})
 
         if self.parsed.custom_headers:
             args_preset.core.merge_custom({"web": {"http_headers": self.parsed.custom_headers}})
@@ -162,13 +166,17 @@ class BBOTArgs:
             except Exception as e:
                 raise BBOTArgumentError(f'Error parsing command-line config option: "{config_arg}": {e}')
 
+        # strict scope
+        if self.parsed.strict_scope:
+            args_preset.core.merge_custom({"scope": {"strict": True}})
+
         return args_preset
 
     def create_parser(self, *args, **kwargs):
         kwargs.update(
-            dict(
-                description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=self.epilog
-            )
+            {
+                "description": "Bighuge BLS OSINT Tool", "formatter_class": argparse.RawTextHelpFormatter, "epilog": self.epilog
+            }
         )
         p = argparse.ArgumentParser(*args, **kwargs)
 
@@ -206,7 +214,7 @@ class BBOTArgs:
             metavar="CONFIG",
             default=[],
         )
-        presets.add_argument("-lp", "--list-presets", action="store_true", help=f"List available presets.")
+        presets.add_argument("-lp", "--list-presets", action="store_true", help="List available presets.")
 
         modules = p.add_argument_group(title="Modules")
         modules.add_argument(
@@ -214,31 +222,31 @@ class BBOTArgs:
             "--modules",
             nargs="+",
             default=[],
-            help=f'Modules to enable. Choices: {",".join(self.preset.module_loader.scan_module_choices)}',
+            help=f'Modules to enable. Choices: {",".join(sorted(self.preset.module_loader.scan_module_choices))}',
             metavar="MODULE",
         )
-        modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.")
+        modules.add_argument("-l", "--list-modules", action="store_true", help="List available modules.")
         modules.add_argument(
             "-lmo", "--list-module-options", action="store_true", help="Show all module config options"
         )
         modules.add_argument(
-            "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE"
+            "-em", "--exclude-modules", nargs="+", default=[], help="Exclude these modules.", metavar="MODULE"
         )
         modules.add_argument(
             "-f",
             "--flags",
             nargs="+",
             default=[],
-            help=f'Enable modules by flag. Choices: {",".join(self.preset.module_loader.flag_choices)}',
+            help=f'Enable modules by flag. Choices: {",".join(sorted(self.preset.module_loader.flag_choices))}',
             metavar="FLAG",
         )
-        modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.")
+        modules.add_argument("-lf", "--list-flags", action="store_true", help="List available flags.")
         modules.add_argument(
             "-rf",
             "--require-flags",
             nargs="+",
             default=[],
-            help=f"Only enable modules with these flags (e.g. -rf passive)",
+            help="Only enable modules with these flags (e.g. -rf passive)",
             metavar="FLAG",
         )
         modules.add_argument(
@@ -246,7 +254,7 @@ class BBOTArgs:
             "--exclude-flags",
             nargs="+",
             default=[],
-            help=f"Disable modules with these flags. (e.g. -ef aggressive)",
+            help="Disable modules with these flags. (e.g. -ef aggressive)",
             metavar="FLAG",
         )
         modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules")
@@ -262,7 +270,12 @@ class BBOTArgs:
             help="Run scan even in the case of condition violations or failed module setups",
         )
         scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt")
-        scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan")
+        scan.add_argument(
+            "--fast-mode",
+            action="store_true",
+            help="Scan only the provided targets as fast as possible, with no extra discovery",
+        )
+        scan.add_argument("--dry-run", action="store_true", help="Abort before executing scan")
         scan.add_argument(
             "--current-preset",
             action="store_true",
@@ -286,7 +299,7 @@ class BBOTArgs:
             "--output-modules",
             nargs="+",
             default=[],
-            help=f'Output module(s). Choices: {",".join(self.preset.module_loader.output_module_choices)}',
+            help=f'Output module(s). Choices: {",".join(sorted(self.preset.module_loader.output_module_choices))}',
             metavar="MODULE",
         )
         output.add_argument("--json", "-j", action="store_true", help="Output scan data in JSON format")
@@ -307,6 +320,7 @@ class BBOTArgs:
 
         misc = p.add_argument_group(title="Misc")
         misc.add_argument("--version", action="store_true", help="show BBOT version and exit")
+        misc.add_argument("--proxy", help="Use this proxy for all HTTP requests", metavar="HTTP_PROXY")
         misc.add_argument(
             "-H",
             "--custom-headers",
@@ -356,6 +370,10 @@ class BBOTArgs:
             custom_headers_dict[k] = v
         self.parsed.custom_headers = custom_headers_dict
 
+        # --fast-mode
+        if self.parsed.fast_mode:
+            self.parsed.preset += ["fast"]
+
     def validate(self):
         # validate config options
         sentinel = object()

bbot/scanner/preset/environ.py

@@ -6,6 +6,9 @@ from pathlib import Path
 from bbot.core.helpers.misc import cpu_architecture, os_platform, os_platform_friendly
 
 
+REQUESTS_PATCHED = False
+
+
 def increase_limit(new_limit):
     try:
         import resource
@@ -62,7 +65,6 @@ omegaconf.OmegaConf.register_new_resolver("env", env_resolver)
 
 
 class BBOTEnviron:
-
     def __init__(self, preset):
         self.preset = preset
 
@@ -120,7 +122,10 @@ class BBOTEnviron:
 
         urllib3.disable_warnings()
         ssl_verify = self.preset.config.get("ssl_verify", False)
-        if not ssl_verify:
+
+        global REQUESTS_PATCHED
+        if not ssl_verify and not REQUESTS_PATCHED:
+            REQUESTS_PATCHED = True
             import requests
             import functools
 

bbot/scanner/preset/path.py

@@ -33,13 +33,16 @@ class PresetPath:
         if "/" in str(filename):
             if filename_path.parent not in paths_to_search:
                 paths_to_search.append(filename_path.parent)
-        log.debug(f"Searching for preset in {paths_to_search}, file candidates: {file_candidates_str}")
+        log.debug(
+            f"Searching for preset in {[str(p) for p in paths_to_search]}, file candidates: {file_candidates_str}"
+        )
         for path in paths_to_search:
             for candidate in file_candidates:
                 for file in path.rglob(candidate):
-                    log.verbose(f'Found preset matching "{filename}" at {file}')
-                    self.add_path(file.parent)
-                    return file.resolve()
+                    if file.is_file():
+                        log.verbose(f'Found preset matching "{filename}" at {file}')
+                        self.add_path(file.parent)
+                        return file.resolve()
         raise ValidationError(
             f'Could not find preset at "{filename}" - file does not exist. Use -lp to list available presets'
         )

bbot/scanner/preset/preset.py

@@ -17,7 +17,7 @@ from bbot.core.helpers.misc import make_table, mkdir, get_closest_match
 log = logging.getLogger("bbot.presets")
 
 
-_preset_cache = dict()
+_preset_cache = {}
 
 
 # cache default presets to prevent having to reload from disk
@@ -47,7 +47,6 @@ class Preset:
         target (Target): Target(s) of scan.
         whitelist (Target): Scan whitelist (by default this is the same as `target`).
         blacklist (Target): Scan blacklist (this takes ultimate precedence).
-        strict_scope (bool): If True, subdomains of targets are not considered to be in-scope.
         helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc.
         output_dir (pathlib.Path): Output directory for scan.
         scan_name (str): Name of scan. Defaults to random value, e.g. "demonic_jimmy".
@@ -87,7 +86,6 @@ class Preset:
         *targets,
         whitelist=None,
         blacklist=None,
-        strict_scope=False,
         modules=None,
         output_modules=None,
         exclude_modules=None,
@@ -117,7 +115,6 @@ class Preset:
             *targets (str): Target(s) to scan. Types supported: hostnames, IPs, CIDRs, emails, open ports.
             whitelist (list, optional): Whitelisted target(s) to scan. Defaults to the same as `targets`.
             blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty.
-            strict_scope (bool, optional): If True, subdomains of targets are not in-scope.
             modules (list[str], optional): List of scan modules to enable for the scan. Defaults to empty list.
             output_modules (list[str], optional): List of output modules to use. Defaults to csv, human, and json.
             exclude_modules (list[str], optional): List of modules to exclude from the scan.
@@ -234,7 +231,6 @@ class Preset:
         self.module_dirs = module_dirs
 
         # target / whitelist / blacklist
-        self.strict_scope = strict_scope
         # these are temporary receptacles until they all get .baked() together
         self._seeds = set(targets if targets else [])
         self._whitelist = set(whitelist) if whitelist else whitelist
@@ -245,7 +241,7 @@ class Preset:
         # "presets" is alias to "include"
         if presets and include:
             raise ValueError(
-                'Cannot use both "presets" and "include" args at the same time (presets is only an alias to include). Please pick only one :)'
+                'Cannot use both "presets" and "include" args at the same time (presets is an alias to include). Please pick one or the other :)'
             )
         if presets and not include:
             include = presets
@@ -274,6 +270,12 @@ class Preset:
             raise ValueError("Cannot access target before preset is baked (use ._seeds instead)")
         return self._target
 
+    @property
+    def seeds(self):
+        if self._seeds is None:
+            raise ValueError("Cannot access target before preset is baked (use ._seeds instead)")
+        return self.target.seeds
+
     @property
     def whitelist(self):
         if self._target is None:
@@ -353,7 +355,9 @@ class Preset:
             else:
                 self._whitelist.update(other._whitelist)
         self._blacklist.update(other._blacklist)
-        self.strict_scope = self.strict_scope or other.strict_scope
+
+        # module dirs
+        self.module_dirs = self.module_dirs.union(other.module_dirs)
 
         # log verbosity
         if other.silent:
@@ -373,6 +377,9 @@ class Preset:
         # misc
         self.force_start = self.force_start | other.force_start
         self._cli = self._cli | other._cli
+        # transfer args
+        if other._args is not None:
+            self._args = other._args
 
     def bake(self, scan=None):
         """
@@ -436,7 +443,7 @@ class Preset:
 
         # disable internal modules if requested
         for internal_module in baked_preset.internal_modules:
-            if baked_preset.config.get(internal_module, True) == False:
+            if baked_preset.config.get(internal_module, True) is False:
                 baked_preset.exclude_modules.add(internal_module)
 
         # enable modules by flag
@@ -531,6 +538,14 @@ class Preset:
     def web_config(self):
         return self.core.config.get("web", {})
 
+    @property
+    def scope_config(self):
+        return self.config.get("scope", {})
+
+    @property
+    def strict_scope(self):
+        return self.scope_config.get("strict", False)
+
     def apply_log_level(self, apply_core=False):
         # silent takes precedence
         if self.silent:
@@ -629,7 +644,6 @@ class Preset:
             debug=preset_dict.get("debug", False),
             silent=preset_dict.get("silent", False),
             config=preset_dict.get("config"),
-            strict_scope=preset_dict.get("strict_scope", False),
             module_dirs=preset_dict.get("module_dirs", []),
             include=list(preset_dict.get("include", [])),
             scan_name=preset_dict.get("scan_name"),
@@ -731,6 +745,9 @@ class Preset:
         """
         preset_dict = {}
 
+        if self.description:
+            preset_dict["description"] = self.description
+
         # config
         if full_config:
             config = self.core.config
@@ -744,19 +761,17 @@ class Preset:
 
         # scope
         if include_target:
-            target = sorted(str(t.data) for t in self.target.seeds)
+            target = sorted(self.target.seeds.inputs)
             whitelist = []
             if self.target.whitelist is not None:
-                whitelist = sorted(str(t.data) for t in self.target.whitelist)
-            blacklist = sorted(str(t.data) for t in self.target.blacklist)
+                whitelist = sorted(self.target.whitelist.inputs)
+            blacklist = sorted(self.target.blacklist.inputs)
             if target:
                 preset_dict["target"] = target
             if whitelist and whitelist != target:
                 preset_dict["whitelist"] = whitelist
             if blacklist:
                 preset_dict["blacklist"] = blacklist
-        if self.strict_scope:
-            preset_dict["strict_scope"] = True
 
         # flags + modules
         if self.require_flags:
@@ -825,7 +840,7 @@ class Preset:
         else:
             raise ValidationError(f'Unknown module type "{module}"')
 
-        if not module in module_choices:
+        if module not in module_choices:
             raise ValidationError(get_closest_match(module, module_choices, msg=f"{module_type} module"))
 
         try:
@@ -838,8 +853,6 @@ class Preset:
 
         if module in self.exclude_modules:
             reason = "the module has been excluded"
-            if raise_error:
-                raise ValidationError(f'Unable to add {module_type} module "{module}" because {reason}')
             return False, reason, {}
 
         module_flags = preloaded.get("flags", [])
@@ -870,21 +883,21 @@ class Preset:
 
         # validate excluded modules
         for excluded_module in self.exclude_modules:
-            if not excluded_module in self.module_loader.all_module_choices:
+            if excluded_module not in self.module_loader.all_module_choices:
                 raise ValidationError(
                     get_closest_match(excluded_module, self.module_loader.all_module_choices, msg="module")
                 )
         # validate excluded flags
         for excluded_flag in self.exclude_flags:
-            if not excluded_flag in self.module_loader.flag_choices:
+            if excluded_flag not in self.module_loader.flag_choices:
                 raise ValidationError(get_closest_match(excluded_flag, self.module_loader.flag_choices, msg="flag"))
         # validate required flags
         for required_flag in self.require_flags:
-            if not required_flag in self.module_loader.flag_choices:
+            if required_flag not in self.module_loader.flag_choices:
                 raise ValidationError(get_closest_match(required_flag, self.module_loader.flag_choices, msg="flag"))
         # validate flags
         for flag in self.flags:
-            if not flag in self.module_loader.flag_choices:
+            if flag not in self.module_loader.flag_choices:
                 raise ValidationError(get_closest_match(flag, self.module_loader.flag_choices, msg="flag"))
 
     @property
@@ -903,7 +916,7 @@ class Preset:
 
         global DEFAULT_PRESETS
         if DEFAULT_PRESETS is None:
-            presets = dict()
+            presets = {}
             for ext in ("yml", "yaml"):
                 for preset_path in PRESET_PATH:
                     # for every yaml file
@@ -954,7 +967,7 @@ class Preset:
         header = ["Preset", "Category", "Description", "# Modules"]
         if include_modules:
             header.append("Modules")
-        for yaml_file, (loaded_preset, category, preset_path, original_file) in self.all_presets.items():
+        for (loaded_preset, category, preset_path, original_file) in self.all_presets.values():
             loaded_preset = loaded_preset.bake()
             num_modules = f"{len(loaded_preset.scan_modules):,}"
             row = [loaded_preset.name, category, loaded_preset.description, num_modules]