bbot 2.2.0.5263rc0__py3-none-any.whl → 2.2.0.5309rc0__py3-none-any.whl

bbot/modules/myssl.py

@@ -17,7 +17,7 @@ class myssl(subdomain_enum):
         url = f"{self.base_url}?domain={self.helpers.quote(query)}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, dict):

bbot/modules/otx.py

@@ -17,10 +17,12 @@ class otx(subdomain_enum):
         url = f"{self.base_url}/api/v1/indicators/domain/{self.helpers.quote(query)}/passive_dns"
         return self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             for entry in j.get("passive_dns", []):
                 subdomain = entry.get("hostname", "")
                 if subdomain:
-                    yield subdomain
+                    results.add(subdomain)
+        return results

bbot/modules/passivetotal.py

@@ -39,6 +39,8 @@ class passivetotal(subdomain_enum_apikey):
         url = f"{self.base_url}/enrichment/subdomains?query={self.helpers.quote(query)}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         for subdomain in r.json().get("subdomains", []):
-            yield f"{subdomain}.{query}"
+            results.add(f"{subdomain}.{query}")
+        return results

bbot/modules/rapiddns.py

@@ -18,11 +18,6 @@ class rapiddns(subdomain_enum):
         response = await self.api_request(url, timeout=self.http_timeout + 10)
         return response
 
-    def parse_results(self, r, query):
-        results = set()
+    async def parse_results(self, r, query):
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)

bbot/modules/securitytrails.py

@@ -26,8 +26,10 @@ class securitytrails(subdomain_enum_apikey):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             for host in j.get("subdomains", []):
-                yield f"{host}.{query}"
+                results.add(f"{host}.{query}")
+        return results

bbot/modules/shodan_dns.py

@@ -22,5 +22,5 @@ class shodan_dns(shodan):
     def make_url(self, query):
         return f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}"
 
-    def parse_results(self, json, query):
+    async def parse_results(self, json, query):
         return [f"{sub}.{query}" for sub in json.get("subdomains", [])]

bbot/modules/subdomaincenter.py

@@ -33,7 +33,7 @@ class subdomaincenter(subdomain_enum):
                 break
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, list):

bbot/modules/templates/subdomain_enum.py

@@ -106,7 +106,7 @@ class subdomain_enum(BaseModule):
                 break
         return ".".join([s for s in query.split(".") if s != "_wildcard"])
 
-    def parse_results(self, r, query=None):
+    async def parse_results(self, r, query=None):
         json = r.json()
         if json:
             for hostname in json:
@@ -123,7 +123,7 @@ class subdomain_enum(BaseModule):
                 self.info(f'Query "{query}" failed (no response)')
                 return []
             try:
-                results = list(parse_fn(response, query))
+                results = list(await parse_fn(response, query))
             except Exception as e:
                 if response:
                     self.info(
@@ -144,7 +144,7 @@ class subdomain_enum(BaseModule):
         agen = self.api_page_iter(url, page_size=self.page_size, **self.api_page_iter_kwargs)
         try:
             async for response in agen:
-                subdomains = self.parse_results(response, query)
+                subdomains = await self.parse_results(response, query)
                 self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
                 if not subdomains:
                     break

bbot/modules/trickest.py

@@ -36,7 +36,7 @@ class Trickest(subdomain_enum_apikey):
         url += "&limit={page_size}&offset={offset}&select=hostname&orderby=hostname"
         return url
 
-    def parse_results(self, j, query):
+    async def parse_results(self, j, query):
         results = j.get("results", [])
         subdomains = set()
         for item in results:

bbot/modules/virustotal.py

@@ -24,11 +24,6 @@ class virustotal(subdomain_enum_apikey):
         kwargs["headers"]["x-apikey"] = self.api_key
         return url, kwargs
 
-    def parse_results(self, r, query):
-        results = set()
+    async def parse_results(self, r, query):
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)

bbot/modules/zoomeye.py

@@ -60,7 +60,7 @@ class zoomeye(subdomain_enum_apikey):
         agen = self.api_page_iter(url)
         try:
             async for j in agen:
-                r = list(self.parse_results(j))
+                r = list(await self.parse_results(j))
                 if r:
                     results.update(set(r))
                 if not r or i >= (self.max_pages - 1):
@@ -70,6 +70,8 @@ class zoomeye(subdomain_enum_apikey):
             agen.aclose()
         return results
 
-    def parse_results(self, r):
+    async def parse_results(self, r):
+        results = set()
         for entry in r.get("list", []):
-            yield entry["name"]
+            results.add(entry["name"])
+        return results

bbot/presets/fast.yml

@@ -0,0 +1,16 @@
+description: Scan only the provided targets as fast as possible - no extra discovery
+
+exclude_modules:
+  - excavate
+
+config:
+  # only scan the exact targets specified
+  scope:
+    strict: true
+  # speed up dns resolution by doing A/AAAA only - not MX/NS/SRV/etc
+  dns:
+    minimal: true
+  # essential speculation only
+  modules:
+    speculate:
+      essential_only: true

bbot/presets/spider.yml

@@ -3,6 +3,10 @@ description: Recursive web spider
 modules:
   - httpx
 
+blacklist:
+  # Prevent spider from invalidating sessions by logging out
+  - "RE:/.*(sign|log)[_-]?out"
+
 config:
   web:
     # how many links to follow in a row

bbot/scanner/manager.py

@@ -38,7 +38,7 @@ class ScanIngress(BaseInterceptModule):
             - It also marks the Scan object as finished with initialization by setting `_finished_init` to True.
         """
         if events is None:
-            events = self.scan.target.events
+            events = self.scan.target.seeds.events
         async with self.scan._acatch(self.init_events), self._task_counter.count(self.init_events):
             sorted_events = sorted(events, key=lambda e: len(e.data))
             for event in [self.scan.root_event] + sorted_events:
@@ -49,7 +49,6 @@ class ScanIngress(BaseInterceptModule):
                     event.parent = self.scan.root_event
                 if event.module is None:
                     event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET")
-                event.add_tag("target")
                 if event != self.scan.root_event:
                     event.discovery_context = f"Scan {self.scan.name} seeded with " + "{event.type}: {event.data}"
                 self.verbose(f"Target: {event}")

bbot/scanner/preset/args.py

@@ -91,7 +91,6 @@ class BBOTArgs:
             *self.parsed.targets,
             whitelist=self.parsed.whitelist,
             blacklist=self.parsed.blacklist,
-            strict_scope=self.parsed.strict_scope,
             name="args_preset",
         )
 
@@ -149,6 +148,9 @@ class BBOTArgs:
         if self.parsed.force:
             args_preset.force_start = self.parsed.force
 
+        if self.parsed.proxy:
+            args_preset.core.merge_custom({"web": {"http_proxy": self.parsed.proxy}})
+
         if self.parsed.custom_headers:
             args_preset.core.merge_custom({"web": {"http_headers": self.parsed.custom_headers}})
 
@@ -165,6 +167,10 @@ class BBOTArgs:
             except Exception as e:
                 raise BBOTArgumentError(f'Error parsing command-line config option: "{config_arg}": {e}')
 
+        # strict scope
+        if self.parsed.strict_scope:
+            args_preset.core.merge_custom({"scope": {"strict": True}})
+
         return args_preset
 
     def create_parser(self, *args, **kwargs):
@@ -217,7 +223,7 @@ class BBOTArgs:
             "--modules",
             nargs="+",
             default=[],
-            help=f'Modules to enable. Choices: {",".join(self.preset.module_loader.scan_module_choices)}',
+            help=f'Modules to enable. Choices: {",".join(sorted(self.preset.module_loader.scan_module_choices))}',
             metavar="MODULE",
         )
         modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.")
@@ -232,7 +238,7 @@ class BBOTArgs:
             "--flags",
             nargs="+",
             default=[],
-            help=f'Enable modules by flag. Choices: {",".join(self.preset.module_loader.flag_choices)}',
+            help=f'Enable modules by flag. Choices: {",".join(sorted(self.preset.module_loader.flag_choices))}',
             metavar="FLAG",
         )
         modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.")
@@ -265,6 +271,11 @@ class BBOTArgs:
             help="Run scan even in the case of condition violations or failed module setups",
         )
         scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt")
+        scan.add_argument(
+            "--fast-mode",
+            action="store_true",
+            help="Scan only the provided targets as fast as possible, with no extra discovery",
+        )
         scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan")
         scan.add_argument(
             "--current-preset",
@@ -289,7 +300,7 @@ class BBOTArgs:
             "--output-modules",
             nargs="+",
             default=[],
-            help=f'Output module(s). Choices: {",".join(self.preset.module_loader.output_module_choices)}',
+            help=f'Output module(s). Choices: {",".join(sorted(self.preset.module_loader.output_module_choices))}',
             metavar="MODULE",
         )
         output.add_argument("--json", "-j", action="store_true", help="Output scan data in JSON format")
@@ -310,6 +321,7 @@ class BBOTArgs:
 
         misc = p.add_argument_group(title="Misc")
         misc.add_argument("--version", action="store_true", help="show BBOT version and exit")
+        misc.add_argument("--proxy", help="Use this proxy for all HTTP requests", metavar="HTTP_PROXY")
         misc.add_argument(
             "-H",
             "--custom-headers",
@@ -359,6 +371,10 @@ class BBOTArgs:
             custom_headers_dict[k] = v
         self.parsed.custom_headers = custom_headers_dict
 
+        # --fast-mode
+        if self.parsed.fast_mode:
+            self.parsed.preset += ["fast"]
+
     def validate(self):
         # validate config options
         sentinel = object()

bbot/scanner/preset/path.py

@@ -33,7 +33,9 @@ class PresetPath:
         if "/" in str(filename):
             if filename_path.parent not in paths_to_search:
                 paths_to_search.append(filename_path.parent)
-        log.debug(f"Searching for preset in {paths_to_search}, file candidates: {file_candidates_str}")
+        log.debug(
+            f"Searching for preset in {[str(p) for p in paths_to_search]}, file candidates: {file_candidates_str}"
+        )
         for path in paths_to_search:
             for candidate in file_candidates:
                 for file in path.rglob(candidate):

bbot/scanner/preset/preset.py

@@ -47,7 +47,6 @@ class Preset:
         target (Target): Target(s) of scan.
         whitelist (Target): Scan whitelist (by default this is the same as `target`).
         blacklist (Target): Scan blacklist (this takes ultimate precedence).
-        strict_scope (bool): If True, subdomains of targets are not considered to be in-scope.
         helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc.
         output_dir (pathlib.Path): Output directory for scan.
         scan_name (str): Name of scan. Defaults to random value, e.g. "demonic_jimmy".
@@ -87,7 +86,6 @@ class Preset:
         *targets,
         whitelist=None,
         blacklist=None,
-        strict_scope=False,
         modules=None,
         output_modules=None,
         exclude_modules=None,
@@ -117,7 +115,6 @@ class Preset:
             *targets (str): Target(s) to scan. Types supported: hostnames, IPs, CIDRs, emails, open ports.
             whitelist (list, optional): Whitelisted target(s) to scan. Defaults to the same as `targets`.
             blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty.
-            strict_scope (bool, optional): If True, subdomains of targets are not in-scope.
             modules (list[str], optional): List of scan modules to enable for the scan. Defaults to empty list.
             output_modules (list[str], optional): List of output modules to use. Defaults to csv, human, and json.
             exclude_modules (list[str], optional): List of modules to exclude from the scan.
@@ -234,7 +231,6 @@ class Preset:
         self.module_dirs = module_dirs
 
         # target / whitelist / blacklist
-        self.strict_scope = strict_scope
         # these are temporary receptacles until they all get .baked() together
         self._seeds = set(targets if targets else [])
         self._whitelist = set(whitelist) if whitelist else whitelist
@@ -245,7 +241,7 @@ class Preset:
         # "presets" is alias to "include"
         if presets and include:
             raise ValueError(
-                'Cannot use both "presets" and "include" args at the same time (presets is only an alias to include). Please pick only one :)'
+                'Cannot use both "presets" and "include" args at the same time (presets is an alias to include). Please pick one or the other :)'
             )
         if presets and not include:
             include = presets
@@ -274,6 +270,12 @@ class Preset:
             raise ValueError("Cannot access target before preset is baked (use ._seeds instead)")
         return self._target
 
+    @property
+    def seeds(self):
+        if self._seeds is None:
+            raise ValueError("Cannot access target before preset is baked (use ._seeds instead)")
+        return self.target.seeds
+
     @property
     def whitelist(self):
         if self._target is None:
@@ -353,7 +355,6 @@ class Preset:
             else:
                 self._whitelist.update(other._whitelist)
         self._blacklist.update(other._blacklist)
-        self.strict_scope = self.strict_scope or other.strict_scope
 
         # module dirs
         self.module_dirs = self.module_dirs.union(other.module_dirs)
@@ -537,6 +538,14 @@ class Preset:
     def web_config(self):
         return self.core.config.get("web", {})
 
+    @property
+    def scope_config(self):
+        return self.config.get("scope", {})
+
+    @property
+    def strict_scope(self):
+        return self.scope_config.get("strict", False)
+
     def apply_log_level(self, apply_core=False):
         # silent takes precedence
         if self.silent:
@@ -635,7 +644,6 @@ class Preset:
             debug=preset_dict.get("debug", False),
             silent=preset_dict.get("silent", False),
             config=preset_dict.get("config"),
-            strict_scope=preset_dict.get("strict_scope", False),
             module_dirs=preset_dict.get("module_dirs", []),
             include=list(preset_dict.get("include", [])),
             scan_name=preset_dict.get("scan_name"),
@@ -753,19 +761,17 @@ class Preset:
 
         # scope
         if include_target:
-            target = sorted(str(t.data) for t in self.target.seeds)
+            target = sorted(self.target.seeds.inputs)
             whitelist = []
             if self.target.whitelist is not None:
-                whitelist = sorted(str(t.data) for t in self.target.whitelist)
-            blacklist = sorted(str(t.data) for t in self.target.blacklist)
+                whitelist = sorted(self.target.whitelist.inputs)
+            blacklist = sorted(self.target.blacklist.inputs)
             if target:
                 preset_dict["target"] = target
             if whitelist and whitelist != target:
                 preset_dict["whitelist"] = whitelist
             if blacklist:
                 preset_dict["blacklist"] = blacklist
-        if self.strict_scope:
-            preset_dict["strict_scope"] = True
 
         # flags + modules
         if self.require_flags:

bbot/scanner/scanner.py

@@ -269,7 +269,7 @@ class Scanner:
                 f.write(self.preset.to_yaml())
 
             # log scan overview
-            start_msg = f"Scan with {len(self.preset.scan_modules):,} modules seeded with {len(self.target):,} targets"
+            start_msg = f"Scan seeded with {len(self.seeds):,} targets"
             details = []
             if self.whitelist != self.target:
                 details.append(f"{len(self.whitelist):,} in whitelist")
@@ -362,7 +362,8 @@ class Scanner:
 
             # distribute seed events
             self.init_events_task = asyncio.create_task(
-                self.ingress_module.init_events(self.target.events), name=f"{self.name}.ingress_module.init_events()"
+                self.ingress_module.init_events(self.target.seeds.events),
+                name=f"{self.name}.ingress_module.init_events()",
             )
 
             # main scan loop
@@ -896,6 +897,10 @@ class Scanner:
     def target(self):
         return self.preset.target
 
+    @property
+    def seeds(self):
+        return self.preset.seeds
+
     @property
     def whitelist(self):
         return self.preset.whitelist