bbot 2.3.0.5370rc0__py3-none-any.whl → 2.3.0.5382rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of bbot might be problematic. Click here for more details.
- bbot/__init__.py +1 -1
- bbot/cli.py +2 -6
- bbot/core/config/files.py +0 -1
- bbot/core/config/logger.py +1 -1
- bbot/core/core.py +1 -1
- bbot/core/event/base.py +13 -16
- bbot/core/helpers/command.py +4 -4
- bbot/core/helpers/depsinstaller/installer.py +5 -5
- bbot/core/helpers/diff.py +7 -7
- bbot/core/helpers/dns/brute.py +1 -1
- bbot/core/helpers/dns/dns.py +1 -2
- bbot/core/helpers/dns/engine.py +4 -6
- bbot/core/helpers/dns/mock.py +0 -1
- bbot/core/helpers/files.py +1 -1
- bbot/core/helpers/helper.py +3 -1
- bbot/core/helpers/interactsh.py +3 -3
- bbot/core/helpers/libmagic.py +0 -1
- bbot/core/helpers/misc.py +11 -11
- bbot/core/helpers/process.py +0 -2
- bbot/core/helpers/regex.py +1 -1
- bbot/core/helpers/regexes.py +3 -3
- bbot/core/helpers/validators.py +1 -2
- bbot/core/helpers/web/client.py +1 -1
- bbot/core/helpers/web/engine.py +1 -2
- bbot/core/helpers/web/web.py +2 -3
- bbot/core/helpers/wordcloud.py +5 -5
- bbot/core/modules.py +21 -22
- bbot/db/sql/models.py +0 -1
- bbot/modules/azure_tenant.py +2 -2
- bbot/modules/baddns.py +0 -2
- bbot/modules/baddns_direct.py +0 -1
- bbot/modules/base.py +16 -16
- bbot/modules/bypass403.py +5 -5
- bbot/modules/c99.py +1 -1
- bbot/modules/columbus.py +1 -1
- bbot/modules/deadly/ffuf.py +8 -8
- bbot/modules/deadly/nuclei.py +1 -1
- bbot/modules/deadly/vhost.py +3 -3
- bbot/modules/dnsbimi.py +1 -1
- bbot/modules/dnsdumpster.py +2 -2
- bbot/modules/dockerhub.py +1 -1
- bbot/modules/dotnetnuke.py +0 -2
- bbot/modules/extractous.py +1 -1
- bbot/modules/filedownload.py +1 -1
- bbot/modules/generic_ssrf.py +3 -3
- bbot/modules/github_workflows.py +1 -1
- bbot/modules/gowitness.py +7 -7
- bbot/modules/host_header.py +5 -5
- bbot/modules/httpx.py +1 -1
- bbot/modules/iis_shortnames.py +6 -6
- bbot/modules/internal/cloudcheck.py +5 -5
- bbot/modules/internal/dnsresolve.py +7 -7
- bbot/modules/internal/excavate.py +23 -26
- bbot/modules/internal/speculate.py +4 -4
- bbot/modules/ipneighbor.py +1 -1
- bbot/modules/jadx.py +1 -1
- bbot/modules/newsletters.py +2 -2
- bbot/modules/output/asset_inventory.py +6 -6
- bbot/modules/output/base.py +1 -1
- bbot/modules/output/csv.py +1 -1
- bbot/modules/output/stdout.py +2 -2
- bbot/modules/paramminer_headers.py +4 -7
- bbot/modules/portscan.py +3 -3
- bbot/modules/report/asn.py +11 -11
- bbot/modules/robots.py +3 -3
- bbot/modules/securitytxt.py +1 -1
- bbot/modules/sitedossier.py +1 -1
- bbot/modules/social.py +1 -1
- bbot/modules/subdomainradar.py +1 -1
- bbot/modules/telerik.py +7 -7
- bbot/modules/templates/bucket.py +1 -1
- bbot/modules/templates/github.py +1 -1
- bbot/modules/templates/shodan.py +1 -1
- bbot/modules/templates/subdomain_enum.py +1 -1
- bbot/modules/templates/webhook.py +1 -1
- bbot/modules/trufflehog.py +2 -2
- bbot/modules/url_manipulation.py +3 -3
- bbot/modules/urlscan.py +1 -1
- bbot/modules/viewdns.py +1 -1
- bbot/modules/wafw00f.py +1 -1
- bbot/scanner/preset/args.py +10 -11
- bbot/scanner/preset/environ.py +0 -1
- bbot/scanner/preset/preset.py +9 -9
- bbot/scanner/scanner.py +17 -17
- bbot/scanner/target.py +1 -1
- bbot/scripts/docs.py +1 -1
- bbot/test/bbot_fixtures.py +1 -1
- bbot/test/conftest.py +1 -1
- bbot/test/run_tests.sh +4 -4
- bbot/test/test_step_1/test_bbot_fastapi.py +2 -2
- bbot/test/test_step_1/test_cli.py +56 -56
- bbot/test/test_step_1/test_dns.py +15 -15
- bbot/test/test_step_1/test_engine.py +17 -17
- bbot/test/test_step_1/test_events.py +22 -22
- bbot/test/test_step_1/test_helpers.py +26 -26
- bbot/test/test_step_1/test_manager_scope_accuracy.py +306 -306
- bbot/test/test_step_1/test_modules_basic.py +52 -53
- bbot/test/test_step_1/test_presets.py +81 -81
- bbot/test/test_step_1/test_regexes.py +5 -5
- bbot/test/test_step_1/test_scan.py +4 -4
- bbot/test/test_step_1/test_target.py +25 -25
- bbot/test/test_step_1/test_web.py +5 -5
- bbot/test/test_step_2/module_tests/base.py +6 -6
- bbot/test/test_step_2/module_tests/test_module_anubisdb.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_asset_inventory.py +0 -1
- bbot/test/test_step_2/module_tests/test_module_azure_realm.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_baddns.py +6 -6
- bbot/test/test_step_2/module_tests/test_module_baddns_direct.py +2 -4
- bbot/test/test_step_2/module_tests/test_module_bevigil.py +4 -4
- bbot/test/test_step_2/module_tests/test_module_binaryedge.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_bucket_azure.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_builtwith.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_c99.py +9 -9
- bbot/test/test_step_2/module_tests/test_module_columbus.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_credshed.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_dehashed.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_digitorus.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_dnsbrute.py +8 -9
- bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py +0 -1
- bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +0 -1
- bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +0 -2
- bbot/test/test_step_2/module_tests/test_module_excavate.py +10 -30
- bbot/test/test_step_2/module_tests/test_module_extractous.py +9 -9
- bbot/test/test_step_2/module_tests/test_module_filedownload.py +14 -14
- bbot/test/test_step_2/module_tests/test_module_git_clone.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_gowitness.py +4 -4
- bbot/test/test_step_2/module_tests/test_module_host_header.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_http.py +4 -4
- bbot/test/test_step_2/module_tests/test_module_httpx.py +7 -7
- bbot/test/test_step_2/module_tests/test_module_leakix.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_myssl.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_neo4j.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_newsletters.py +6 -6
- bbot/test/test_step_2/module_tests/test_module_ntlm.py +7 -7
- bbot/test/test_step_2/module_tests/test_module_oauth.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_otx.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_paramminer_cookies.py +1 -2
- bbot/test/test_step_2/module_tests/test_module_paramminer_getparams.py +0 -6
- bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py +2 -9
- bbot/test/test_step_2/module_tests/test_module_portscan.py +3 -4
- bbot/test/test_step_2/module_tests/test_module_postgres.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_rapiddns.py +9 -9
- bbot/test/test_step_2/module_tests/test_module_sitedossier.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_smuggler.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_speculate.py +2 -6
- bbot/test/test_step_2/module_tests/test_module_splunk.py +4 -4
- bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_subdomains.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_trufflehog.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_wayback.py +1 -1
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/METADATA +2 -2
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/RECORD +157 -157
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/LICENSE +0 -0
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/WHEEL +0 -0
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/entry_points.txt +0 -0
bbot/__init__.py
CHANGED
bbot/cli.py
CHANGED
|
@@ -29,7 +29,6 @@ scan_name = ""
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
async def _main():
|
|
32
|
-
|
|
33
32
|
import asyncio
|
|
34
33
|
import traceback
|
|
35
34
|
from contextlib import suppress
|
|
@@ -45,7 +44,6 @@ async def _main():
|
|
|
45
44
|
global scan_name
|
|
46
45
|
|
|
47
46
|
try:
|
|
48
|
-
|
|
49
47
|
# start by creating a default scan preset
|
|
50
48
|
preset = Preset(_log=True, name="bbot_cli_main")
|
|
51
49
|
# parse command line arguments and merge into preset
|
|
@@ -81,7 +79,6 @@ async def _main():
|
|
|
81
79
|
|
|
82
80
|
# if we're listing modules or their options
|
|
83
81
|
if options.list_modules or options.list_module_options:
|
|
84
|
-
|
|
85
82
|
# if no modules or flags are specified, enable everything
|
|
86
83
|
if not (options.modules or options.output_modules or options.flags):
|
|
87
84
|
for module, preloaded in preset.module_loader.preloaded().items():
|
|
@@ -133,8 +130,8 @@ async def _main():
|
|
|
133
130
|
]
|
|
134
131
|
if deadly_modules and not options.allow_deadly:
|
|
135
132
|
log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}")
|
|
136
|
-
log.hugewarning(
|
|
137
|
-
log.hugewarning(
|
|
133
|
+
log.hugewarning("Deadly modules are highly intrusive")
|
|
134
|
+
log.hugewarning("Please specify --allow-deadly to continue")
|
|
138
135
|
return False
|
|
139
136
|
|
|
140
137
|
# --current-preset
|
|
@@ -172,7 +169,6 @@ async def _main():
|
|
|
172
169
|
log.trace(f"Command: {' '.join(sys.argv)}")
|
|
173
170
|
|
|
174
171
|
if sys.stdin.isatty():
|
|
175
|
-
|
|
176
172
|
# warn if any targets belong directly to a cloud provider
|
|
177
173
|
for event in scan.target.seeds.events:
|
|
178
174
|
if event.type == "DNS_NAME":
|
bbot/core/config/files.py
CHANGED
|
@@ -10,7 +10,6 @@ bbot_code_dir = Path(__file__).parent.parent.parent
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class BBOTConfigFiles:
|
|
13
|
-
|
|
14
13
|
config_dir = (Path.home() / ".config" / "bbot").resolve()
|
|
15
14
|
defaults_filename = (bbot_code_dir / "defaults.yml").resolve()
|
|
16
15
|
config_filename = (config_dir / "bbot.yml").resolve()
|
bbot/core/config/logger.py
CHANGED
|
@@ -68,7 +68,7 @@ class BBOTLogger:
|
|
|
68
68
|
self.listener = None
|
|
69
69
|
|
|
70
70
|
# if we haven't set up logging yet, do it now
|
|
71
|
-
if
|
|
71
|
+
if "_BBOT_LOGGING_SETUP" not in os.environ:
|
|
72
72
|
os.environ["_BBOT_LOGGING_SETUP"] = "1"
|
|
73
73
|
self.queue = multiprocessing.Queue()
|
|
74
74
|
self.setup_queue_handler()
|
bbot/core/core.py
CHANGED
|
@@ -106,7 +106,7 @@ class BBOTCore:
|
|
|
106
106
|
if DEFAULT_CONFIG is None:
|
|
107
107
|
self.default_config = self.files_config.get_default_config()
|
|
108
108
|
# ensure bbot home dir
|
|
109
|
-
if
|
|
109
|
+
if "home" not in self.default_config:
|
|
110
110
|
self.default_config["home"] = "~/.bbot"
|
|
111
111
|
return DEFAULT_CONFIG
|
|
112
112
|
|
bbot/core/event/base.py
CHANGED
|
@@ -175,8 +175,8 @@ class BaseEvent:
|
|
|
175
175
|
self._scope_distance = None
|
|
176
176
|
self._module_priority = None
|
|
177
177
|
self._resolved_hosts = set()
|
|
178
|
-
self.dns_children =
|
|
179
|
-
self.raw_dns_records =
|
|
178
|
+
self.dns_children = {}
|
|
179
|
+
self.raw_dns_records = {}
|
|
180
180
|
self._discovery_context = ""
|
|
181
181
|
self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}")
|
|
182
182
|
self.web_spider_distance = 0
|
|
@@ -203,7 +203,7 @@ class BaseEvent:
|
|
|
203
203
|
# self.scan holds the instantiated scan object (for helpers, etc.)
|
|
204
204
|
self.scan = scan
|
|
205
205
|
if (not self.scan) and (not self._dummy):
|
|
206
|
-
raise ValidationError(
|
|
206
|
+
raise ValidationError("Must specify scan")
|
|
207
207
|
# self.scans holds a list of scan IDs from scans that encountered this event
|
|
208
208
|
self.scans = []
|
|
209
209
|
if scans is not None:
|
|
@@ -222,7 +222,7 @@ class BaseEvent:
|
|
|
222
222
|
|
|
223
223
|
self.parent = parent
|
|
224
224
|
if (not self.parent) and (not self._dummy):
|
|
225
|
-
raise ValidationError(
|
|
225
|
+
raise ValidationError("Must specify event parent")
|
|
226
226
|
|
|
227
227
|
if tags is not None:
|
|
228
228
|
for tag in tags:
|
|
@@ -301,9 +301,9 @@ class BaseEvent:
|
|
|
301
301
|
The purpose of internal events is to enable speculative/explorative discovery without cluttering
|
|
302
302
|
the console with irrelevant or uninteresting events.
|
|
303
303
|
"""
|
|
304
|
-
if not
|
|
304
|
+
if value not in (True, False):
|
|
305
305
|
raise ValueError(f'"internal" must be boolean, not {type(value)}')
|
|
306
|
-
if value
|
|
306
|
+
if value is True:
|
|
307
307
|
self.add_tag("internal")
|
|
308
308
|
else:
|
|
309
309
|
self.remove_tag("internal")
|
|
@@ -769,7 +769,7 @@ class BaseEvent:
|
|
|
769
769
|
Returns:
|
|
770
770
|
dict: JSON-serializable dictionary representation of the event object.
|
|
771
771
|
"""
|
|
772
|
-
j =
|
|
772
|
+
j = {}
|
|
773
773
|
# type, ID, scope description
|
|
774
774
|
for i in ("type", "id", "uuid", "scope_description", "netloc"):
|
|
775
775
|
v = getattr(self, i, "")
|
|
@@ -1013,12 +1013,12 @@ class ClosestHostEvent(DictHostEvent):
|
|
|
1013
1013
|
if not self.host:
|
|
1014
1014
|
for parent in self.get_parents(include_self=True):
|
|
1015
1015
|
# inherit closest URL
|
|
1016
|
-
if
|
|
1016
|
+
if "url" not in self.data:
|
|
1017
1017
|
parent_url = getattr(parent, "parsed_url", None)
|
|
1018
1018
|
if parent_url is not None:
|
|
1019
1019
|
self.data["url"] = parent_url.geturl()
|
|
1020
1020
|
# inherit closest path
|
|
1021
|
-
if
|
|
1021
|
+
if "path" not in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE":
|
|
1022
1022
|
parent_path = parent.data.get("path", None)
|
|
1023
1023
|
if parent_path is not None:
|
|
1024
1024
|
self.data["path"] = parent_path
|
|
@@ -1180,7 +1180,6 @@ class URL_UNVERIFIED(BaseEvent):
|
|
|
1180
1180
|
self.num_redirects = getattr(self.parent, "num_redirects", 0)
|
|
1181
1181
|
|
|
1182
1182
|
def _data_id(self):
|
|
1183
|
-
|
|
1184
1183
|
data = super()._data_id()
|
|
1185
1184
|
|
|
1186
1185
|
# remove the querystring for URL/URL_UNVERIFIED events, because we will conditionally add it back in (based on settings)
|
|
@@ -1228,7 +1227,7 @@ class URL_UNVERIFIED(BaseEvent):
|
|
|
1228
1227
|
|
|
1229
1228
|
def add_tag(self, tag):
|
|
1230
1229
|
host_same_as_parent = self.parent and self.host == self.parent.host
|
|
1231
|
-
if tag == "spider-danger" and host_same_as_parent and
|
|
1230
|
+
if tag == "spider-danger" and host_same_as_parent and "spider-danger" not in self.tags:
|
|
1232
1231
|
# increment the web spider distance
|
|
1233
1232
|
if self.type == "URL_UNVERIFIED":
|
|
1234
1233
|
self.web_spider_distance += 1
|
|
@@ -1250,7 +1249,7 @@ class URL_UNVERIFIED(BaseEvent):
|
|
|
1250
1249
|
|
|
1251
1250
|
def _words(self):
|
|
1252
1251
|
first_elem = self.parsed_url.path.lstrip("/").split("/")[0]
|
|
1253
|
-
if
|
|
1252
|
+
if "." not in first_elem:
|
|
1254
1253
|
return extract_words(first_elem)
|
|
1255
1254
|
return set()
|
|
1256
1255
|
|
|
@@ -1267,7 +1266,6 @@ class URL_UNVERIFIED(BaseEvent):
|
|
|
1267
1266
|
|
|
1268
1267
|
|
|
1269
1268
|
class URL(URL_UNVERIFIED):
|
|
1270
|
-
|
|
1271
1269
|
def __init__(self, *args, **kwargs):
|
|
1272
1270
|
super().__init__(*args, **kwargs)
|
|
1273
1271
|
|
|
@@ -1279,7 +1277,7 @@ class URL(URL_UNVERIFIED):
|
|
|
1279
1277
|
@property
|
|
1280
1278
|
def resolved_hosts(self):
|
|
1281
1279
|
# TODO: remove this when we rip out httpx
|
|
1282
|
-
return
|
|
1280
|
+
return {".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-")}
|
|
1283
1281
|
|
|
1284
1282
|
@property
|
|
1285
1283
|
def pretty_string(self):
|
|
@@ -1309,7 +1307,6 @@ class URL_HINT(URL_UNVERIFIED):
|
|
|
1309
1307
|
|
|
1310
1308
|
|
|
1311
1309
|
class WEB_PARAMETER(DictHostEvent):
|
|
1312
|
-
|
|
1313
1310
|
def _data_id(self):
|
|
1314
1311
|
# dedupe by url:name:param_type
|
|
1315
1312
|
url = self.data.get("url", "")
|
|
@@ -1668,7 +1665,7 @@ def make_event(
|
|
|
1668
1665
|
event.parent = parent
|
|
1669
1666
|
if context is not None:
|
|
1670
1667
|
event.discovery_context = context
|
|
1671
|
-
if internal
|
|
1668
|
+
if internal is True:
|
|
1672
1669
|
event.internal = True
|
|
1673
1670
|
if tags:
|
|
1674
1671
|
event.tags = tags.union(event.tags)
|
bbot/core/helpers/command.py
CHANGED
|
@@ -269,11 +269,11 @@ def _prepare_command_kwargs(self, command, kwargs):
|
|
|
269
269
|
(['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)})
|
|
270
270
|
"""
|
|
271
271
|
# limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout)
|
|
272
|
-
if
|
|
272
|
+
if "limit" not in kwargs:
|
|
273
273
|
kwargs["limit"] = 1024 * 1024 * 100
|
|
274
|
-
if
|
|
274
|
+
if "stdout" not in kwargs:
|
|
275
275
|
kwargs["stdout"] = asyncio.subprocess.PIPE
|
|
276
|
-
if
|
|
276
|
+
if "stderr" not in kwargs:
|
|
277
277
|
kwargs["stderr"] = asyncio.subprocess.PIPE
|
|
278
278
|
sudo = kwargs.pop("sudo", False)
|
|
279
279
|
|
|
@@ -286,7 +286,7 @@ def _prepare_command_kwargs(self, command, kwargs):
|
|
|
286
286
|
|
|
287
287
|
# use full path of binary, if not already specified
|
|
288
288
|
binary = command[0]
|
|
289
|
-
if
|
|
289
|
+
if "/" not in binary:
|
|
290
290
|
binary_full_path = which(binary)
|
|
291
291
|
if binary_full_path is None:
|
|
292
292
|
raise SubprocessError(f'Command "{binary}" was not found')
|
|
@@ -97,11 +97,11 @@ class DepsInstaller:
|
|
|
97
97
|
or self.deps_behavior == "force_install"
|
|
98
98
|
):
|
|
99
99
|
if not notified:
|
|
100
|
-
log.hugeinfo(
|
|
100
|
+
log.hugeinfo("Installing module dependencies. Please be patient, this may take a while.")
|
|
101
101
|
notified = True
|
|
102
102
|
log.verbose(f'Installing dependencies for module "{m}"')
|
|
103
103
|
# get sudo access if we need it
|
|
104
|
-
if preloaded.get("sudo", False)
|
|
104
|
+
if preloaded.get("sudo", False) is True:
|
|
105
105
|
self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.')
|
|
106
106
|
success = await self.install_module(m)
|
|
107
107
|
self.setup_status[module_hash] = success
|
|
@@ -159,7 +159,7 @@ class DepsInstaller:
|
|
|
159
159
|
deps_common = preloaded["deps"]["common"]
|
|
160
160
|
if deps_common:
|
|
161
161
|
for dep_common in deps_common:
|
|
162
|
-
if self.setup_status.get(dep_common, False)
|
|
162
|
+
if self.setup_status.get(dep_common, False) is True:
|
|
163
163
|
log.debug(
|
|
164
164
|
f'Skipping installation of dependency "{dep_common}" for module "{module}" since it is already installed'
|
|
165
165
|
)
|
|
@@ -244,7 +244,7 @@ class DepsInstaller:
|
|
|
244
244
|
if success:
|
|
245
245
|
log.info(f"Successfully ran {len(commands):,} shell commands")
|
|
246
246
|
else:
|
|
247
|
-
log.warning(
|
|
247
|
+
log.warning("Failed to run shell dependencies")
|
|
248
248
|
return success
|
|
249
249
|
|
|
250
250
|
def tasks(self, module, tasks):
|
|
@@ -310,7 +310,7 @@ class DepsInstaller:
|
|
|
310
310
|
return success, err
|
|
311
311
|
|
|
312
312
|
def read_setup_status(self):
|
|
313
|
-
setup_status =
|
|
313
|
+
setup_status = {}
|
|
314
314
|
if self.setup_status_cache.is_file():
|
|
315
315
|
with open(self.setup_status_cache) as f:
|
|
316
316
|
with suppress(Exception):
|
bbot/core/helpers/diff.py
CHANGED
|
@@ -101,7 +101,7 @@ class HttpCompare:
|
|
|
101
101
|
ddiff = DeepDiff(baseline_1_json, baseline_2_json, ignore_order=True, view="tree")
|
|
102
102
|
self.ddiff_filters = []
|
|
103
103
|
|
|
104
|
-
for k
|
|
104
|
+
for k in ddiff.keys():
|
|
105
105
|
for x in list(ddiff[k]):
|
|
106
106
|
log.debug(f"Added {k} filter for path: {x.path()}")
|
|
107
107
|
self.ddiff_filters.append(x.path())
|
|
@@ -140,7 +140,7 @@ class HttpCompare:
|
|
|
140
140
|
|
|
141
141
|
ddiff = DeepDiff(headers_1, headers_2, ignore_order=True, view="tree")
|
|
142
142
|
|
|
143
|
-
for k
|
|
143
|
+
for k in ddiff.keys():
|
|
144
144
|
for x in list(ddiff[k]):
|
|
145
145
|
try:
|
|
146
146
|
header_value = str(x).split("'")[1]
|
|
@@ -183,7 +183,7 @@ class HttpCompare:
|
|
|
183
183
|
|
|
184
184
|
await self._baseline()
|
|
185
185
|
|
|
186
|
-
if timeout
|
|
186
|
+
if timeout is None:
|
|
187
187
|
timeout = self.timeout
|
|
188
188
|
|
|
189
189
|
reflection = False
|
|
@@ -238,11 +238,11 @@ class HttpCompare:
|
|
|
238
238
|
|
|
239
239
|
different_headers = self.compare_headers(self.baseline.headers, subject_response.headers)
|
|
240
240
|
if different_headers:
|
|
241
|
-
log.debug(
|
|
241
|
+
log.debug("headers were different, no match")
|
|
242
242
|
diff_reasons.append("header")
|
|
243
243
|
|
|
244
|
-
if self.compare_body(self.baseline_json, subject_json)
|
|
245
|
-
log.debug(
|
|
244
|
+
if self.compare_body(self.baseline_json, subject_json) is False:
|
|
245
|
+
log.debug("difference in HTML body, no match")
|
|
246
246
|
|
|
247
247
|
diff_reasons.append("body")
|
|
248
248
|
|
|
@@ -275,6 +275,6 @@ class HttpCompare:
|
|
|
275
275
|
)
|
|
276
276
|
|
|
277
277
|
# if a nonsense header "caused" a difference, we need to abort. We also need to abort if our canary was reflected
|
|
278
|
-
if match
|
|
278
|
+
if match is False or reflection is True:
|
|
279
279
|
return False
|
|
280
280
|
return True
|
bbot/core/helpers/dns/brute.py
CHANGED
|
@@ -164,7 +164,7 @@ class DNSBrute:
|
|
|
164
164
|
for i in range(0, max(0, n - 5)):
|
|
165
165
|
d = delimiters[i % len(delimiters)]
|
|
166
166
|
l = lengths[i % len(lengths)]
|
|
167
|
-
segments =
|
|
167
|
+
segments = [random.choice(self.devops_mutations) for _ in range(l)]
|
|
168
168
|
segments.append(self.parent_helper.rand_string(length=8, digits=False))
|
|
169
169
|
subdomain = d.join(segments)
|
|
170
170
|
yield subdomain
|
bbot/core/helpers/dns/dns.py
CHANGED
|
@@ -16,7 +16,6 @@ log = logging.getLogger("bbot.core.helpers.dns")
|
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
class DNSHelper(EngineClient):
|
|
19
|
-
|
|
20
19
|
SERVER_CLASS = DNSEngine
|
|
21
20
|
ERROR_CLASS = DNSError
|
|
22
21
|
|
|
@@ -179,7 +178,7 @@ class DNSHelper(EngineClient):
|
|
|
179
178
|
|
|
180
179
|
host = clean_dns_record(host)
|
|
181
180
|
# skip check if it's an IP or a plain hostname
|
|
182
|
-
if is_ip(host) or
|
|
181
|
+
if is_ip(host) or "." not in host:
|
|
183
182
|
return False
|
|
184
183
|
|
|
185
184
|
# skip if query isn't a dns name
|
bbot/core/helpers/dns/engine.py
CHANGED
|
@@ -24,7 +24,6 @@ all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"]
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
class DNSEngine(EngineServer):
|
|
27
|
-
|
|
28
27
|
CMDS = {
|
|
29
28
|
0: "resolve",
|
|
30
29
|
1: "resolve_raw",
|
|
@@ -55,7 +54,7 @@ class DNSEngine(EngineServer):
|
|
|
55
54
|
dns_omit_queries = self.dns_config.get("omit_queries", None)
|
|
56
55
|
if not dns_omit_queries:
|
|
57
56
|
dns_omit_queries = []
|
|
58
|
-
self.dns_omit_queries =
|
|
57
|
+
self.dns_omit_queries = {}
|
|
59
58
|
for d in dns_omit_queries:
|
|
60
59
|
d = d.split(":")
|
|
61
60
|
if len(d) == 2:
|
|
@@ -73,7 +72,7 @@ class DNSEngine(EngineServer):
|
|
|
73
72
|
self.wildcard_ignore = []
|
|
74
73
|
self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore])
|
|
75
74
|
self.wildcard_tests = self.dns_config.get("wildcard_tests", 5)
|
|
76
|
-
self._wildcard_cache =
|
|
75
|
+
self._wildcard_cache = {}
|
|
77
76
|
# since wildcard detection takes some time, This is to prevent multiple
|
|
78
77
|
# modules from kicking off wildcard detection for the same domain at the same time
|
|
79
78
|
self._wildcard_lock = NamedLock()
|
|
@@ -83,7 +82,7 @@ class DNSEngine(EngineServer):
|
|
|
83
82
|
self._last_connectivity_warning = time.time()
|
|
84
83
|
# keeps track of warnings issued for wildcard detection to prevent duplicate warnings
|
|
85
84
|
self._dns_warnings = set()
|
|
86
|
-
self._errors =
|
|
85
|
+
self._errors = {}
|
|
87
86
|
self._debug = self.dns_config.get("debug", False)
|
|
88
87
|
self._dns_cache = LRUCache(maxsize=10000)
|
|
89
88
|
|
|
@@ -476,7 +475,6 @@ class DNSEngine(EngineServer):
|
|
|
476
475
|
# for every parent domain, starting with the shortest
|
|
477
476
|
parents = list(domain_parents(query))
|
|
478
477
|
for parent in parents[::-1]:
|
|
479
|
-
|
|
480
478
|
# check if the parent domain is set up with wildcards
|
|
481
479
|
wildcard_results = await self.is_wildcard_domain(parent, rdtypes_to_check)
|
|
482
480
|
|
|
@@ -640,7 +638,7 @@ class DNSEngine(EngineServer):
|
|
|
640
638
|
self._last_dns_success = time.time()
|
|
641
639
|
return True
|
|
642
640
|
if time.time() - self._last_connectivity_warning > interval:
|
|
643
|
-
self.log.warning(
|
|
641
|
+
self.log.warning("DNS queries are failing, please check your internet connection")
|
|
644
642
|
self._last_connectivity_warning = time.time()
|
|
645
643
|
self._errors.clear()
|
|
646
644
|
return False
|
bbot/core/helpers/dns/mock.py
CHANGED
bbot/core/helpers/files.py
CHANGED
|
@@ -83,7 +83,7 @@ def _feed_pipe(self, pipe, content, text=True):
|
|
|
83
83
|
for c in content:
|
|
84
84
|
p.write(decode_fn(c) + newline)
|
|
85
85
|
except BrokenPipeError:
|
|
86
|
-
log.debug(
|
|
86
|
+
log.debug("Broken pipe in _feed_pipe()")
|
|
87
87
|
except ValueError:
|
|
88
88
|
log.debug(f"Error _feed_pipe(): {traceback.format_exc()}")
|
|
89
89
|
except KeyboardInterrupt:
|
bbot/core/helpers/helper.py
CHANGED
|
@@ -153,7 +153,9 @@ class ConfigAwareHelper:
|
|
|
153
153
|
return self.temp_dir / filename
|
|
154
154
|
|
|
155
155
|
def clean_old_scans(self):
|
|
156
|
-
_filter
|
|
156
|
+
def _filter(x):
|
|
157
|
+
return x.is_dir() and self.regexes.scan_name_regex.match(x.name)
|
|
158
|
+
|
|
157
159
|
self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter)
|
|
158
160
|
|
|
159
161
|
def make_target(self, *targets, **kwargs):
|
bbot/core/helpers/interactsh.py
CHANGED
|
@@ -155,7 +155,7 @@ class Interactsh:
|
|
|
155
155
|
break
|
|
156
156
|
|
|
157
157
|
if not self.server:
|
|
158
|
-
raise InteractshError(
|
|
158
|
+
raise InteractshError("Failed to register with an interactsh server")
|
|
159
159
|
|
|
160
160
|
log.info(
|
|
161
161
|
f"Successfully registered to interactsh server {self.server} with correlation_id {self.correlation_id} [{self.domain}]"
|
|
@@ -181,7 +181,7 @@ class Interactsh:
|
|
|
181
181
|
>>> await interactsh_client.deregister()
|
|
182
182
|
"""
|
|
183
183
|
if not self.server or not self.correlation_id or not self.secret:
|
|
184
|
-
raise InteractshError(
|
|
184
|
+
raise InteractshError("Missing required information to deregister")
|
|
185
185
|
|
|
186
186
|
headers = {}
|
|
187
187
|
if self.token:
|
|
@@ -226,7 +226,7 @@ class Interactsh:
|
|
|
226
226
|
]
|
|
227
227
|
"""
|
|
228
228
|
if not self.server or not self.correlation_id or not self.secret:
|
|
229
|
-
raise InteractshError(
|
|
229
|
+
raise InteractshError("Missing required information to poll")
|
|
230
230
|
|
|
231
231
|
headers = {}
|
|
232
232
|
if self.token:
|
bbot/core/helpers/libmagic.py
CHANGED
bbot/core/helpers/misc.py
CHANGED
|
@@ -391,7 +391,7 @@ def url_parents(u):
|
|
|
391
391
|
parent_list = []
|
|
392
392
|
while 1:
|
|
393
393
|
parent = parent_url(u)
|
|
394
|
-
if parent
|
|
394
|
+
if parent is None:
|
|
395
395
|
return parent_list
|
|
396
396
|
elif parent not in parent_list:
|
|
397
397
|
parent_list.append(parent)
|
|
@@ -512,7 +512,7 @@ def domain_stem(domain):
|
|
|
512
512
|
- Utilizes the `tldextract` function for domain parsing.
|
|
513
513
|
"""
|
|
514
514
|
parsed = tldextract(str(domain))
|
|
515
|
-
return
|
|
515
|
+
return ".".join(parsed.subdomain.split(".") + parsed.domain.split(".")).strip(".")
|
|
516
516
|
|
|
517
517
|
|
|
518
518
|
def ip_network_parents(i, include_self=False):
|
|
@@ -921,12 +921,12 @@ def extract_params_xml(xml_data, compare_mode="getparam"):
|
|
|
921
921
|
|
|
922
922
|
# Define valid characters for each mode based on RFCs
|
|
923
923
|
valid_chars_dict = {
|
|
924
|
-
"header":
|
|
924
|
+
"header": {
|
|
925
925
|
chr(c) for c in range(33, 127) if chr(c) in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"
|
|
926
|
-
|
|
927
|
-
"getparam":
|
|
928
|
-
"postparam":
|
|
929
|
-
"cookie":
|
|
926
|
+
},
|
|
927
|
+
"getparam": {chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="},
|
|
928
|
+
"postparam": {chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="},
|
|
929
|
+
"cookie": {chr(c) for c in range(33, 127) if chr(c) not in '()<>@,;:"/[]?={} \t'},
|
|
930
930
|
}
|
|
931
931
|
|
|
932
932
|
|
|
@@ -1148,7 +1148,7 @@ def chain_lists(
|
|
|
1148
1148
|
"""
|
|
1149
1149
|
if isinstance(l, str):
|
|
1150
1150
|
l = [l]
|
|
1151
|
-
final_list =
|
|
1151
|
+
final_list = {}
|
|
1152
1152
|
for entry in l:
|
|
1153
1153
|
for s in split_regex.split(entry):
|
|
1154
1154
|
f = s.strip()
|
|
@@ -1345,7 +1345,7 @@ def search_dict_by_key(key, d):
|
|
|
1345
1345
|
if isinstance(d, dict):
|
|
1346
1346
|
if key in d:
|
|
1347
1347
|
yield d[key]
|
|
1348
|
-
for
|
|
1348
|
+
for v in d.values():
|
|
1349
1349
|
yield from search_dict_by_key(key, v)
|
|
1350
1350
|
elif isinstance(d, list):
|
|
1351
1351
|
for v in d:
|
|
@@ -1412,7 +1412,7 @@ def search_dict_values(d, *regexes):
|
|
|
1412
1412
|
results.add(h)
|
|
1413
1413
|
yield result
|
|
1414
1414
|
elif isinstance(d, dict):
|
|
1415
|
-
for
|
|
1415
|
+
for v in d.values():
|
|
1416
1416
|
yield from search_dict_values(v, *regexes)
|
|
1417
1417
|
elif isinstance(d, list):
|
|
1418
1418
|
for v in d:
|
|
@@ -2397,7 +2397,7 @@ def in_exception_chain(e, exc_types):
|
|
|
2397
2397
|
... if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):
|
|
2398
2398
|
... raise
|
|
2399
2399
|
"""
|
|
2400
|
-
return any(
|
|
2400
|
+
return any(isinstance(_, exc_types) for _ in get_exception_chain(e))
|
|
2401
2401
|
|
|
2402
2402
|
|
|
2403
2403
|
def get_traceback_details(e):
|
bbot/core/helpers/process.py
CHANGED
|
@@ -7,7 +7,6 @@ from .misc import in_exception_chain
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class BBOTThread(threading.Thread):
|
|
10
|
-
|
|
11
10
|
default_name = "default bbot thread"
|
|
12
11
|
|
|
13
12
|
def __init__(self, *args, **kwargs):
|
|
@@ -24,7 +23,6 @@ class BBOTThread(threading.Thread):
|
|
|
24
23
|
|
|
25
24
|
|
|
26
25
|
class BBOTProcess(SpawnProcess):
|
|
27
|
-
|
|
28
26
|
default_name = "bbot process pool"
|
|
29
27
|
|
|
30
28
|
def __init__(self, *args, **kwargs):
|
bbot/core/helpers/regex.py
CHANGED
|
@@ -41,7 +41,7 @@ class RegexHelper:
|
|
|
41
41
|
"""
|
|
42
42
|
if not isinstance(compiled_regexes, dict):
|
|
43
43
|
raise ValueError('compiled_regexes must be a dictionary like this: {"regex_name": <compiled_regex>}')
|
|
44
|
-
for
|
|
44
|
+
for v in compiled_regexes.values():
|
|
45
45
|
self.ensure_compiled_regex(v)
|
|
46
46
|
|
|
47
47
|
tasks = {}
|
bbot/core/helpers/regexes.py
CHANGED
|
@@ -36,7 +36,7 @@ _ip_range_regexes = (
|
|
|
36
36
|
_ipv4_regex + r"\/[0-9]{1,2}",
|
|
37
37
|
_ipv6_regex + r"\/[0-9]{1,3}",
|
|
38
38
|
)
|
|
39
|
-
ip_range_regexes =
|
|
39
|
+
ip_range_regexes = [re.compile(r, re.I) for r in _ip_range_regexes]
|
|
40
40
|
|
|
41
41
|
# dns names with periods
|
|
42
42
|
_dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
|
|
@@ -64,14 +64,14 @@ _open_port_regexes = (
|
|
|
64
64
|
_hostname_regex + r":[0-9]{1,5}",
|
|
65
65
|
r"\[" + _ipv6_regex + r"\]:[0-9]{1,5}",
|
|
66
66
|
)
|
|
67
|
-
open_port_regexes =
|
|
67
|
+
open_port_regexes = [re.compile(r, re.I) for r in _open_port_regexes]
|
|
68
68
|
|
|
69
69
|
_url_regexes = (
|
|
70
70
|
r"https?://" + _dns_name_regex + r"(?::[0-9]{1,5})?(?:(?:/|\?).*)?",
|
|
71
71
|
r"https?://" + _hostname_regex + r"(?::[0-9]{1,5})?(?:(?:/|\?).*)?",
|
|
72
72
|
r"https?://\[" + _ipv6_regex + r"\](?::[0-9]{1,5})?(?:(?:/|\?).*)?",
|
|
73
73
|
)
|
|
74
|
-
url_regexes =
|
|
74
|
+
url_regexes = [re.compile(r, re.I) for r in _url_regexes]
|
|
75
75
|
|
|
76
76
|
_double_slash_regex = r"/{2,}"
|
|
77
77
|
double_slash_regex = re.compile(_double_slash_regex)
|
bbot/core/helpers/validators.py
CHANGED
|
@@ -132,7 +132,7 @@ def validate_host(host: Union[str, ipaddress.IPv4Address, ipaddress.IPv6Address]
|
|
|
132
132
|
@validator
|
|
133
133
|
def validate_severity(severity: str):
|
|
134
134
|
severity = str(severity).strip().upper()
|
|
135
|
-
if not
|
|
135
|
+
if severity not in ("UNKNOWN", "INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"):
|
|
136
136
|
raise ValueError(f"Invalid severity: {severity}")
|
|
137
137
|
return severity
|
|
138
138
|
|
|
@@ -299,7 +299,6 @@ def is_email(email):
|
|
|
299
299
|
|
|
300
300
|
|
|
301
301
|
class Validators:
|
|
302
|
-
|
|
303
302
|
def __init__(self, parent_helper):
|
|
304
303
|
self.parent_helper = parent_helper
|
|
305
304
|
|
bbot/core/helpers/web/client.py
CHANGED
bbot/core/helpers/web/engine.py
CHANGED
|
@@ -14,7 +14,6 @@ log = logging.getLogger("bbot.core.helpers.web.engine")
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class HTTPEngine(EngineServer):
|
|
17
|
-
|
|
18
17
|
CMDS = {
|
|
19
18
|
0: "request",
|
|
20
19
|
1: "request_batch",
|
|
@@ -138,7 +137,7 @@ class HTTPEngine(EngineServer):
|
|
|
138
137
|
if max_size is not None:
|
|
139
138
|
max_size = human_to_bytes(max_size)
|
|
140
139
|
kwargs["follow_redirects"] = follow_redirects
|
|
141
|
-
if
|
|
140
|
+
if "method" not in kwargs:
|
|
142
141
|
kwargs["method"] = "GET"
|
|
143
142
|
try:
|
|
144
143
|
total_size = 0
|
bbot/core/helpers/web/web.py
CHANGED
|
@@ -19,7 +19,6 @@ log = logging.getLogger("bbot.core.helpers.web")
|
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
class WebHelper(EngineClient):
|
|
22
|
-
|
|
23
22
|
SERVER_CLASS = HTTPEngine
|
|
24
23
|
ERROR_CLASS = WebError
|
|
25
24
|
|
|
@@ -262,7 +261,7 @@ class WebHelper(EngineClient):
|
|
|
262
261
|
"""
|
|
263
262
|
if not path:
|
|
264
263
|
raise WordlistError(f"Invalid wordlist: {path}")
|
|
265
|
-
if
|
|
264
|
+
if "cache_hrs" not in kwargs:
|
|
266
265
|
kwargs["cache_hrs"] = 720
|
|
267
266
|
if self.parent_helper.is_url(path):
|
|
268
267
|
filename = await self.download(str(path), **kwargs)
|
|
@@ -351,7 +350,7 @@ class WebHelper(EngineClient):
|
|
|
351
350
|
headers[hk] = hv
|
|
352
351
|
|
|
353
352
|
# add the timeout
|
|
354
|
-
if
|
|
353
|
+
if "timeout" not in kwargs:
|
|
355
354
|
timeout = http_timeout
|
|
356
355
|
|
|
357
356
|
curl_command.append("-m")
|