bbot 2.3.0.5370rc0__py3-none-any.whl → 2.3.0.5382rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of bbot might be problematic. Click here for more details.
- bbot/__init__.py +1 -1
- bbot/cli.py +2 -6
- bbot/core/config/files.py +0 -1
- bbot/core/config/logger.py +1 -1
- bbot/core/core.py +1 -1
- bbot/core/event/base.py +13 -16
- bbot/core/helpers/command.py +4 -4
- bbot/core/helpers/depsinstaller/installer.py +5 -5
- bbot/core/helpers/diff.py +7 -7
- bbot/core/helpers/dns/brute.py +1 -1
- bbot/core/helpers/dns/dns.py +1 -2
- bbot/core/helpers/dns/engine.py +4 -6
- bbot/core/helpers/dns/mock.py +0 -1
- bbot/core/helpers/files.py +1 -1
- bbot/core/helpers/helper.py +3 -1
- bbot/core/helpers/interactsh.py +3 -3
- bbot/core/helpers/libmagic.py +0 -1
- bbot/core/helpers/misc.py +11 -11
- bbot/core/helpers/process.py +0 -2
- bbot/core/helpers/regex.py +1 -1
- bbot/core/helpers/regexes.py +3 -3
- bbot/core/helpers/validators.py +1 -2
- bbot/core/helpers/web/client.py +1 -1
- bbot/core/helpers/web/engine.py +1 -2
- bbot/core/helpers/web/web.py +2 -3
- bbot/core/helpers/wordcloud.py +5 -5
- bbot/core/modules.py +21 -22
- bbot/db/sql/models.py +0 -1
- bbot/modules/azure_tenant.py +2 -2
- bbot/modules/baddns.py +0 -2
- bbot/modules/baddns_direct.py +0 -1
- bbot/modules/base.py +16 -16
- bbot/modules/bypass403.py +5 -5
- bbot/modules/c99.py +1 -1
- bbot/modules/columbus.py +1 -1
- bbot/modules/deadly/ffuf.py +8 -8
- bbot/modules/deadly/nuclei.py +1 -1
- bbot/modules/deadly/vhost.py +3 -3
- bbot/modules/dnsbimi.py +1 -1
- bbot/modules/dnsdumpster.py +2 -2
- bbot/modules/dockerhub.py +1 -1
- bbot/modules/dotnetnuke.py +0 -2
- bbot/modules/extractous.py +1 -1
- bbot/modules/filedownload.py +1 -1
- bbot/modules/generic_ssrf.py +3 -3
- bbot/modules/github_workflows.py +1 -1
- bbot/modules/gowitness.py +7 -7
- bbot/modules/host_header.py +5 -5
- bbot/modules/httpx.py +1 -1
- bbot/modules/iis_shortnames.py +6 -6
- bbot/modules/internal/cloudcheck.py +5 -5
- bbot/modules/internal/dnsresolve.py +7 -7
- bbot/modules/internal/excavate.py +23 -26
- bbot/modules/internal/speculate.py +4 -4
- bbot/modules/ipneighbor.py +1 -1
- bbot/modules/jadx.py +1 -1
- bbot/modules/newsletters.py +2 -2
- bbot/modules/output/asset_inventory.py +6 -6
- bbot/modules/output/base.py +1 -1
- bbot/modules/output/csv.py +1 -1
- bbot/modules/output/stdout.py +2 -2
- bbot/modules/paramminer_headers.py +4 -7
- bbot/modules/portscan.py +3 -3
- bbot/modules/report/asn.py +11 -11
- bbot/modules/robots.py +3 -3
- bbot/modules/securitytxt.py +1 -1
- bbot/modules/sitedossier.py +1 -1
- bbot/modules/social.py +1 -1
- bbot/modules/subdomainradar.py +1 -1
- bbot/modules/telerik.py +7 -7
- bbot/modules/templates/bucket.py +1 -1
- bbot/modules/templates/github.py +1 -1
- bbot/modules/templates/shodan.py +1 -1
- bbot/modules/templates/subdomain_enum.py +1 -1
- bbot/modules/templates/webhook.py +1 -1
- bbot/modules/trufflehog.py +2 -2
- bbot/modules/url_manipulation.py +3 -3
- bbot/modules/urlscan.py +1 -1
- bbot/modules/viewdns.py +1 -1
- bbot/modules/wafw00f.py +1 -1
- bbot/scanner/preset/args.py +10 -11
- bbot/scanner/preset/environ.py +0 -1
- bbot/scanner/preset/preset.py +9 -9
- bbot/scanner/scanner.py +17 -17
- bbot/scanner/target.py +1 -1
- bbot/scripts/docs.py +1 -1
- bbot/test/bbot_fixtures.py +1 -1
- bbot/test/conftest.py +1 -1
- bbot/test/run_tests.sh +4 -4
- bbot/test/test_step_1/test_bbot_fastapi.py +2 -2
- bbot/test/test_step_1/test_cli.py +56 -56
- bbot/test/test_step_1/test_dns.py +15 -15
- bbot/test/test_step_1/test_engine.py +17 -17
- bbot/test/test_step_1/test_events.py +22 -22
- bbot/test/test_step_1/test_helpers.py +26 -26
- bbot/test/test_step_1/test_manager_scope_accuracy.py +306 -306
- bbot/test/test_step_1/test_modules_basic.py +52 -53
- bbot/test/test_step_1/test_presets.py +81 -81
- bbot/test/test_step_1/test_regexes.py +5 -5
- bbot/test/test_step_1/test_scan.py +4 -4
- bbot/test/test_step_1/test_target.py +25 -25
- bbot/test/test_step_1/test_web.py +5 -5
- bbot/test/test_step_2/module_tests/base.py +6 -6
- bbot/test/test_step_2/module_tests/test_module_anubisdb.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_asset_inventory.py +0 -1
- bbot/test/test_step_2/module_tests/test_module_azure_realm.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_baddns.py +6 -6
- bbot/test/test_step_2/module_tests/test_module_baddns_direct.py +2 -4
- bbot/test/test_step_2/module_tests/test_module_bevigil.py +4 -4
- bbot/test/test_step_2/module_tests/test_module_binaryedge.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_bucket_azure.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_builtwith.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_c99.py +9 -9
- bbot/test/test_step_2/module_tests/test_module_columbus.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_credshed.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_dehashed.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_digitorus.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_dnsbrute.py +8 -9
- bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py +0 -1
- bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +0 -1
- bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +0 -2
- bbot/test/test_step_2/module_tests/test_module_excavate.py +10 -30
- bbot/test/test_step_2/module_tests/test_module_extractous.py +9 -9
- bbot/test/test_step_2/module_tests/test_module_filedownload.py +14 -14
- bbot/test/test_step_2/module_tests/test_module_git_clone.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_gowitness.py +4 -4
- bbot/test/test_step_2/module_tests/test_module_host_header.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_http.py +4 -4
- bbot/test/test_step_2/module_tests/test_module_httpx.py +7 -7
- bbot/test/test_step_2/module_tests/test_module_leakix.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_myssl.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_neo4j.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_newsletters.py +6 -6
- bbot/test/test_step_2/module_tests/test_module_ntlm.py +7 -7
- bbot/test/test_step_2/module_tests/test_module_oauth.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_otx.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_paramminer_cookies.py +1 -2
- bbot/test/test_step_2/module_tests/test_module_paramminer_getparams.py +0 -6
- bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py +2 -9
- bbot/test/test_step_2/module_tests/test_module_portscan.py +3 -4
- bbot/test/test_step_2/module_tests/test_module_postgres.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_rapiddns.py +9 -9
- bbot/test/test_step_2/module_tests/test_module_sitedossier.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_smuggler.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_speculate.py +2 -6
- bbot/test/test_step_2/module_tests/test_module_splunk.py +4 -4
- bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_subdomains.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_trufflehog.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_wayback.py +1 -1
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/METADATA +2 -2
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/RECORD +157 -157
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/LICENSE +0 -0
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/WHEEL +0 -0
- {bbot-2.3.0.5370rc0.dist-info → bbot-2.3.0.5382rc0.dist-info}/entry_points.txt +0 -0
bbot/modules/portscan.py
CHANGED
|
@@ -99,7 +99,7 @@ class portscan(BaseModule):
|
|
|
99
99
|
return False, "Masscan failed to run"
|
|
100
100
|
returncode = getattr(ipv6_result, "returncode", 0)
|
|
101
101
|
if returncode and "failed to detect IPv6 address" in ipv6_result.stderr:
|
|
102
|
-
self.warning(
|
|
102
|
+
self.warning("It looks like you are not set up for IPv6. IPv6 targets will not be scanned.")
|
|
103
103
|
self.ipv6_support = False
|
|
104
104
|
return True
|
|
105
105
|
|
|
@@ -109,7 +109,7 @@ class portscan(BaseModule):
|
|
|
109
109
|
self.scanned_initial_targets = True
|
|
110
110
|
events = set(events)
|
|
111
111
|
events.update(
|
|
112
|
-
|
|
112
|
+
{e for e in self.scan.target.seeds.events if e.type in ("DNS_NAME", "IP_ADDRESS", "IP_RANGE")}
|
|
113
113
|
)
|
|
114
114
|
|
|
115
115
|
# ping scan
|
|
@@ -334,7 +334,7 @@ class portscan(BaseModule):
|
|
|
334
334
|
if "FAIL" in s:
|
|
335
335
|
self.warning(s)
|
|
336
336
|
self.warning(
|
|
337
|
-
|
|
337
|
+
'Masscan failed to detect interface. Recommend passing "adapter_ip", "adapter_mac", and "router_mac" config options to portscan module.'
|
|
338
338
|
)
|
|
339
339
|
else:
|
|
340
340
|
self.verbose(s)
|
bbot/modules/report/asn.py
CHANGED
|
@@ -38,7 +38,7 @@ class asn(BaseReportModule):
|
|
|
38
38
|
|
|
39
39
|
async def handle_event(self, event):
|
|
40
40
|
host = event.host
|
|
41
|
-
if self.cache_get(host)
|
|
41
|
+
if self.cache_get(host) is False:
|
|
42
42
|
asns, source = await self.get_asn(host)
|
|
43
43
|
if not asns:
|
|
44
44
|
self.cache_put(self.unknown_asn)
|
|
@@ -96,7 +96,7 @@ class asn(BaseReportModule):
|
|
|
96
96
|
for p in self.helpers.ip_network_parents(ip):
|
|
97
97
|
try:
|
|
98
98
|
self.asn_counts[p] += 1
|
|
99
|
-
if ret
|
|
99
|
+
if ret is False:
|
|
100
100
|
ret = p
|
|
101
101
|
except KeyError:
|
|
102
102
|
continue
|
|
@@ -112,7 +112,7 @@ class asn(BaseReportModule):
|
|
|
112
112
|
for i, source in enumerate(list(self.sources)):
|
|
113
113
|
get_asn_fn = getattr(self, f"get_asn_{source}")
|
|
114
114
|
res = await get_asn_fn(ip)
|
|
115
|
-
if res
|
|
115
|
+
if res is False:
|
|
116
116
|
# demote the current source to lowest priority since it just failed
|
|
117
117
|
self.sources.append(self.sources.pop(i))
|
|
118
118
|
self.verbose(f"Failed to contact {source}, retrying")
|
|
@@ -125,7 +125,7 @@ class asn(BaseReportModule):
|
|
|
125
125
|
url = f"https://stat.ripe.net/data/network-info/data.json?resource={ip}"
|
|
126
126
|
response = await self.get_url(url, "ASN")
|
|
127
127
|
asns = []
|
|
128
|
-
if response
|
|
128
|
+
if response is False:
|
|
129
129
|
return False
|
|
130
130
|
data = response.get("data", {})
|
|
131
131
|
if not data:
|
|
@@ -138,7 +138,7 @@ class asn(BaseReportModule):
|
|
|
138
138
|
asn_numbers = []
|
|
139
139
|
for number in asn_numbers:
|
|
140
140
|
asn = await self.get_asn_metadata_ripe(number)
|
|
141
|
-
if asn
|
|
141
|
+
if asn is False:
|
|
142
142
|
return False
|
|
143
143
|
asn["subnet"] = prefix
|
|
144
144
|
asns.append(asn)
|
|
@@ -155,7 +155,7 @@ class asn(BaseReportModule):
|
|
|
155
155
|
}
|
|
156
156
|
url = f"https://stat.ripe.net/data/whois/data.json?resource={asn_number}"
|
|
157
157
|
response = await self.get_url(url, "ASN Metadata", cache=True)
|
|
158
|
-
if response
|
|
158
|
+
if response is False:
|
|
159
159
|
return False
|
|
160
160
|
data = response.get("data", {})
|
|
161
161
|
if not data:
|
|
@@ -187,7 +187,7 @@ class asn(BaseReportModule):
|
|
|
187
187
|
data = await self.get_url(url, "ASN")
|
|
188
188
|
asns = []
|
|
189
189
|
asns_tried = set()
|
|
190
|
-
if data
|
|
190
|
+
if data is False:
|
|
191
191
|
return False
|
|
192
192
|
data = data.get("data", {})
|
|
193
193
|
prefixes = data.get("prefixes", [])
|
|
@@ -201,13 +201,13 @@ class asn(BaseReportModule):
|
|
|
201
201
|
description = details.get("description") or prefix.get("description") or ""
|
|
202
202
|
country = details.get("country_code") or prefix.get("country_code") or ""
|
|
203
203
|
emails = []
|
|
204
|
-
if not
|
|
204
|
+
if asn not in asns_tried:
|
|
205
205
|
emails = await self.get_emails_bgpview(asn)
|
|
206
|
-
if emails
|
|
206
|
+
if emails is False:
|
|
207
207
|
return False
|
|
208
208
|
asns_tried.add(asn)
|
|
209
209
|
asns.append(
|
|
210
|
-
|
|
210
|
+
{"asn": asn, "subnet": subnet, "name": name, "description": description, "country": country, "emails": emails}
|
|
211
211
|
)
|
|
212
212
|
if not asns:
|
|
213
213
|
self.debug(f'No results for "{ip}"')
|
|
@@ -217,7 +217,7 @@ class asn(BaseReportModule):
|
|
|
217
217
|
contacts = []
|
|
218
218
|
url = f"https://api.bgpview.io/asn/{asn}"
|
|
219
219
|
data = await self.get_url(url, "ASN metadata", cache=True)
|
|
220
|
-
if data
|
|
220
|
+
if data is False:
|
|
221
221
|
return False
|
|
222
222
|
data = data.get("data", {})
|
|
223
223
|
if not data:
|
bbot/modules/robots.py
CHANGED
|
@@ -33,14 +33,14 @@ class robots(BaseModule):
|
|
|
33
33
|
for l in lines:
|
|
34
34
|
if len(l) > 0:
|
|
35
35
|
split_l = l.split(": ")
|
|
36
|
-
if (split_l[0].lower() == "allow" and self.config.get("include_allow")
|
|
37
|
-
split_l[0].lower() == "disallow" and self.config.get("include_disallow")
|
|
36
|
+
if (split_l[0].lower() == "allow" and self.config.get("include_allow") is True) or (
|
|
37
|
+
split_l[0].lower() == "disallow" and self.config.get("include_disallow") is True
|
|
38
38
|
):
|
|
39
39
|
unverified_url = f"{host}{split_l[1].lstrip('/')}".replace(
|
|
40
40
|
"*", self.helpers.rand_string(4)
|
|
41
41
|
)
|
|
42
42
|
|
|
43
|
-
elif split_l[0].lower() == "sitemap" and self.config.get("include_sitemap")
|
|
43
|
+
elif split_l[0].lower() == "sitemap" and self.config.get("include_sitemap") is True:
|
|
44
44
|
unverified_url = split_l[1]
|
|
45
45
|
else:
|
|
46
46
|
continue
|
bbot/modules/securitytxt.py
CHANGED
|
@@ -121,7 +121,7 @@ class securitytxt(BaseModule):
|
|
|
121
121
|
start, end = match.span()
|
|
122
122
|
found_url = v[start:end]
|
|
123
123
|
|
|
124
|
-
if found_url != url and self._urls
|
|
124
|
+
if found_url != url and self._urls is True:
|
|
125
125
|
await self.emit_event(found_url, "URL_UNVERIFIED", parent=event, tags=tags)
|
|
126
126
|
|
|
127
127
|
|
bbot/modules/sitedossier.py
CHANGED
bbot/modules/social.py
CHANGED
|
@@ -45,7 +45,7 @@ class social(BaseModule):
|
|
|
45
45
|
url = f"https://{url}"
|
|
46
46
|
event_data = {"platform": platform, "url": url, "profile_name": profile_name}
|
|
47
47
|
# only emit if the same event isn't already in the parent chain
|
|
48
|
-
if not any(
|
|
48
|
+
if not any(e.type == "SOCIAL" and e.data == event_data for e in event.get_parents()):
|
|
49
49
|
social_event = self.make_event(
|
|
50
50
|
event_data,
|
|
51
51
|
"SOCIAL",
|
bbot/modules/subdomainradar.py
CHANGED
|
@@ -46,7 +46,7 @@ class SubdomainRadar(subdomain_enum_apikey):
|
|
|
46
46
|
try:
|
|
47
47
|
j = response.json()
|
|
48
48
|
except Exception:
|
|
49
|
-
return False,
|
|
49
|
+
return False, "Failed to get enumerators: failed to parse response as JSON"
|
|
50
50
|
for group in j:
|
|
51
51
|
group_name = group.get("name", "").strip().lower()
|
|
52
52
|
if group_name:
|
bbot/modules/telerik.py
CHANGED
|
@@ -174,7 +174,7 @@ class telerik(BaseModule):
|
|
|
174
174
|
result, _ = await self.test_detector(event.data, webresource)
|
|
175
175
|
if result:
|
|
176
176
|
if "RadAsyncUpload handler is registered succesfully" in result.text:
|
|
177
|
-
self.debug(
|
|
177
|
+
self.debug("Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)")
|
|
178
178
|
|
|
179
179
|
probe_data = {
|
|
180
180
|
"rauPostData": (
|
|
@@ -216,7 +216,7 @@ class telerik(BaseModule):
|
|
|
216
216
|
event,
|
|
217
217
|
context=f"{{module}} scanned {event.data} and identified {{event.type}}: Telerik RAU AXD Handler",
|
|
218
218
|
)
|
|
219
|
-
if self.config.get("exploit_RAU_crypto")
|
|
219
|
+
if self.config.get("exploit_RAU_crypto") is True:
|
|
220
220
|
hostname = urlparse(event.data).netloc
|
|
221
221
|
if hostname not in self.RAUConfirmed:
|
|
222
222
|
self.RAUConfirmed.append(hostname)
|
|
@@ -270,7 +270,7 @@ class telerik(BaseModule):
|
|
|
270
270
|
else:
|
|
271
271
|
if "Cannot deserialize dialog parameters" in response.text:
|
|
272
272
|
self.debug(f"Detected Telerik UI instance ({dh})")
|
|
273
|
-
description =
|
|
273
|
+
description = "Telerik DialogHandler detected"
|
|
274
274
|
await self.emit_event(
|
|
275
275
|
{"host": str(event.host), "url": f"{event.data}{dh}", "description": description},
|
|
276
276
|
"FINDING",
|
|
@@ -289,8 +289,8 @@ class telerik(BaseModule):
|
|
|
289
289
|
self.debug(validate_result)
|
|
290
290
|
validate_status_code = getattr(validate_result, "status_code", 0)
|
|
291
291
|
if validate_status_code not in (0, 500):
|
|
292
|
-
self.debug(
|
|
293
|
-
description =
|
|
292
|
+
self.debug("Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)")
|
|
293
|
+
description = "Telerik SpellCheckHandler detected"
|
|
294
294
|
await self.emit_event(
|
|
295
295
|
{
|
|
296
296
|
"host": str(event.host),
|
|
@@ -334,7 +334,7 @@ class telerik(BaseModule):
|
|
|
334
334
|
},
|
|
335
335
|
"FINDING",
|
|
336
336
|
event,
|
|
337
|
-
context=
|
|
337
|
+
context="{module} searched HTTP_RESPONSE and identified {event.type}: Telerik ChartImage AXD Handler",
|
|
338
338
|
)
|
|
339
339
|
elif '"_serializedConfiguration":"' in resp_body:
|
|
340
340
|
await self.emit_event(
|
|
@@ -345,7 +345,7 @@ class telerik(BaseModule):
|
|
|
345
345
|
},
|
|
346
346
|
"FINDING",
|
|
347
347
|
event,
|
|
348
|
-
context=
|
|
348
|
+
context="{module} searched HTTP_RESPONSE and identified {event.type}: Telerik AsyncUpload",
|
|
349
349
|
)
|
|
350
350
|
|
|
351
351
|
# Check for RAD Controls in URL
|
bbot/modules/templates/bucket.py
CHANGED
|
@@ -159,7 +159,7 @@ class bucket_template(BaseModule):
|
|
|
159
159
|
valid = self.cloud_helper.is_valid_bucket_name(bucket_name)
|
|
160
160
|
if valid and not self.helpers.is_ip(bucket_name):
|
|
161
161
|
bucket_hash = hash(bucket_name)
|
|
162
|
-
if not
|
|
162
|
+
if bucket_hash not in self.buckets_tried:
|
|
163
163
|
self.buckets_tried.add(bucket_hash)
|
|
164
164
|
return True
|
|
165
165
|
return False
|
bbot/modules/templates/github.py
CHANGED
bbot/modules/templates/shodan.py
CHANGED
|
@@ -155,7 +155,7 @@ class subdomain_enum(BaseModule):
|
|
|
155
155
|
async def _is_wildcard(self, query):
|
|
156
156
|
rdtypes = ("A", "AAAA", "CNAME")
|
|
157
157
|
if self.helpers.is_dns_name(query):
|
|
158
|
-
for
|
|
158
|
+
for wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query, rdtypes=rdtypes)).values():
|
|
159
159
|
if any(t in wildcard_rdtypes for t in rdtypes):
|
|
160
160
|
return True
|
|
161
161
|
return False
|
|
@@ -60,7 +60,7 @@ class WebhookOutputModule(BaseOutputModule):
|
|
|
60
60
|
async def filter_event(self, event):
|
|
61
61
|
if event.type == "VULNERABILITY":
|
|
62
62
|
severity = event.data.get("severity", "UNKNOWN")
|
|
63
|
-
if not
|
|
63
|
+
if severity not in self.allowed_severities:
|
|
64
64
|
return False, f"{severity} is below min_severity threshold"
|
|
65
65
|
return True
|
|
66
66
|
|
bbot/modules/trufflehog.py
CHANGED
|
@@ -13,7 +13,7 @@ class trufflehog(BaseModule):
|
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
options = {
|
|
16
|
-
"version": "3.84.
|
|
16
|
+
"version": "3.84.1",
|
|
17
17
|
"config": "",
|
|
18
18
|
"only_verified": True,
|
|
19
19
|
"concurrency": 8,
|
|
@@ -51,7 +51,7 @@ class trufflehog(BaseModule):
|
|
|
51
51
|
self.github_token = ""
|
|
52
52
|
if self.deleted_forks:
|
|
53
53
|
self.warning(
|
|
54
|
-
|
|
54
|
+
"Deleted forks is enabled. Scanning for deleted forks is slooooooowwwww. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours."
|
|
55
55
|
)
|
|
56
56
|
for module_name in ("github", "github_codesearch", "github_org", "git_clone"):
|
|
57
57
|
module_config = self.scan.config.get("modules", {}).get(module_name, {})
|
bbot/modules/url_manipulation.py
CHANGED
|
@@ -69,11 +69,11 @@ class url_manipulation(BaseModule):
|
|
|
69
69
|
|
|
70
70
|
if subject_response:
|
|
71
71
|
subject_content = "".join([str(x) for x in subject_response.headers])
|
|
72
|
-
if subject_response.text
|
|
72
|
+
if subject_response.text is not None:
|
|
73
73
|
subject_content += subject_response.text
|
|
74
74
|
|
|
75
75
|
if self.rand_string not in subject_content:
|
|
76
|
-
if match
|
|
76
|
+
if match is False:
|
|
77
77
|
if str(subject_response.status_code).startswith("2"):
|
|
78
78
|
if "body" in reasons:
|
|
79
79
|
reported_signature = f"Modified URL: {sig[1]}"
|
|
@@ -98,7 +98,7 @@ class url_manipulation(BaseModule):
|
|
|
98
98
|
return False
|
|
99
99
|
|
|
100
100
|
def format_signature(self, sig, event):
|
|
101
|
-
if sig[2]
|
|
101
|
+
if sig[2] is True:
|
|
102
102
|
cleaned_path = event.parsed_url.path.strip("/")
|
|
103
103
|
else:
|
|
104
104
|
cleaned_path = event.parsed_url.path.lstrip("/")
|
bbot/modules/urlscan.py
CHANGED
bbot/modules/viewdns.py
CHANGED
|
@@ -48,7 +48,7 @@ class viewdns(BaseModule):
|
|
|
48
48
|
|
|
49
49
|
html = self.helpers.beautifulsoup(content, "html.parser")
|
|
50
50
|
if html is False:
|
|
51
|
-
self.debug(
|
|
51
|
+
self.debug("BeautifulSoup returned False")
|
|
52
52
|
return results
|
|
53
53
|
found = set()
|
|
54
54
|
for table_row in html.findAll("tr"):
|
bbot/modules/wafw00f.py
CHANGED
|
@@ -52,7 +52,7 @@ class wafw00f(BaseModule):
|
|
|
52
52
|
context=f"{{module}} scanned {url} and identified {{event.type}}: {waf}",
|
|
53
53
|
)
|
|
54
54
|
else:
|
|
55
|
-
if self.config.get("generic_detect")
|
|
55
|
+
if self.config.get("generic_detect") is True:
|
|
56
56
|
generic = await self.helpers.run_in_executor(WW.genericdetect)
|
|
57
57
|
if generic:
|
|
58
58
|
waf = "generic detection"
|
bbot/scanner/preset/args.py
CHANGED
|
@@ -10,7 +10,6 @@ log = logging.getLogger("bbot.presets.args")
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class BBOTArgs:
|
|
13
|
-
|
|
14
13
|
# module config options to exclude from validation
|
|
15
14
|
exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|module_threads)$")
|
|
16
15
|
|
|
@@ -175,9 +174,9 @@ class BBOTArgs:
|
|
|
175
174
|
|
|
176
175
|
def create_parser(self, *args, **kwargs):
|
|
177
176
|
kwargs.update(
|
|
178
|
-
|
|
179
|
-
description
|
|
180
|
-
|
|
177
|
+
{
|
|
178
|
+
"description": "Bighuge BLS OSINT Tool", "formatter_class": argparse.RawTextHelpFormatter, "epilog": self.epilog
|
|
179
|
+
}
|
|
181
180
|
)
|
|
182
181
|
p = argparse.ArgumentParser(*args, **kwargs)
|
|
183
182
|
|
|
@@ -215,7 +214,7 @@ class BBOTArgs:
|
|
|
215
214
|
metavar="CONFIG",
|
|
216
215
|
default=[],
|
|
217
216
|
)
|
|
218
|
-
presets.add_argument("-lp", "--list-presets", action="store_true", help=
|
|
217
|
+
presets.add_argument("-lp", "--list-presets", action="store_true", help="List available presets.")
|
|
219
218
|
|
|
220
219
|
modules = p.add_argument_group(title="Modules")
|
|
221
220
|
modules.add_argument(
|
|
@@ -226,12 +225,12 @@ class BBOTArgs:
|
|
|
226
225
|
help=f'Modules to enable. Choices: {",".join(sorted(self.preset.module_loader.scan_module_choices))}',
|
|
227
226
|
metavar="MODULE",
|
|
228
227
|
)
|
|
229
|
-
modules.add_argument("-l", "--list-modules", action="store_true", help=
|
|
228
|
+
modules.add_argument("-l", "--list-modules", action="store_true", help="List available modules.")
|
|
230
229
|
modules.add_argument(
|
|
231
230
|
"-lmo", "--list-module-options", action="store_true", help="Show all module config options"
|
|
232
231
|
)
|
|
233
232
|
modules.add_argument(
|
|
234
|
-
"-em", "--exclude-modules", nargs="+", default=[], help=
|
|
233
|
+
"-em", "--exclude-modules", nargs="+", default=[], help="Exclude these modules.", metavar="MODULE"
|
|
235
234
|
)
|
|
236
235
|
modules.add_argument(
|
|
237
236
|
"-f",
|
|
@@ -241,13 +240,13 @@ class BBOTArgs:
|
|
|
241
240
|
help=f'Enable modules by flag. Choices: {",".join(sorted(self.preset.module_loader.flag_choices))}',
|
|
242
241
|
metavar="FLAG",
|
|
243
242
|
)
|
|
244
|
-
modules.add_argument("-lf", "--list-flags", action="store_true", help=
|
|
243
|
+
modules.add_argument("-lf", "--list-flags", action="store_true", help="List available flags.")
|
|
245
244
|
modules.add_argument(
|
|
246
245
|
"-rf",
|
|
247
246
|
"--require-flags",
|
|
248
247
|
nargs="+",
|
|
249
248
|
default=[],
|
|
250
|
-
help=
|
|
249
|
+
help="Only enable modules with these flags (e.g. -rf passive)",
|
|
251
250
|
metavar="FLAG",
|
|
252
251
|
)
|
|
253
252
|
modules.add_argument(
|
|
@@ -255,7 +254,7 @@ class BBOTArgs:
|
|
|
255
254
|
"--exclude-flags",
|
|
256
255
|
nargs="+",
|
|
257
256
|
default=[],
|
|
258
|
-
help=
|
|
257
|
+
help="Disable modules with these flags. (e.g. -ef aggressive)",
|
|
259
258
|
metavar="FLAG",
|
|
260
259
|
)
|
|
261
260
|
modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules")
|
|
@@ -276,7 +275,7 @@ class BBOTArgs:
|
|
|
276
275
|
action="store_true",
|
|
277
276
|
help="Scan only the provided targets as fast as possible, with no extra discovery",
|
|
278
277
|
)
|
|
279
|
-
scan.add_argument("--dry-run", action="store_true", help=
|
|
278
|
+
scan.add_argument("--dry-run", action="store_true", help="Abort before executing scan")
|
|
280
279
|
scan.add_argument(
|
|
281
280
|
"--current-preset",
|
|
282
281
|
action="store_true",
|
bbot/scanner/preset/environ.py
CHANGED
bbot/scanner/preset/preset.py
CHANGED
|
@@ -17,7 +17,7 @@ from bbot.core.helpers.misc import make_table, mkdir, get_closest_match
|
|
|
17
17
|
log = logging.getLogger("bbot.presets")
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
_preset_cache =
|
|
20
|
+
_preset_cache = {}
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
# cache default presets to prevent having to reload from disk
|
|
@@ -443,7 +443,7 @@ class Preset:
|
|
|
443
443
|
|
|
444
444
|
# disable internal modules if requested
|
|
445
445
|
for internal_module in baked_preset.internal_modules:
|
|
446
|
-
if baked_preset.config.get(internal_module, True)
|
|
446
|
+
if baked_preset.config.get(internal_module, True) is False:
|
|
447
447
|
baked_preset.exclude_modules.add(internal_module)
|
|
448
448
|
|
|
449
449
|
# enable modules by flag
|
|
@@ -840,7 +840,7 @@ class Preset:
|
|
|
840
840
|
else:
|
|
841
841
|
raise ValidationError(f'Unknown module type "{module}"')
|
|
842
842
|
|
|
843
|
-
if not
|
|
843
|
+
if module not in module_choices:
|
|
844
844
|
raise ValidationError(get_closest_match(module, module_choices, msg=f"{module_type} module"))
|
|
845
845
|
|
|
846
846
|
try:
|
|
@@ -883,21 +883,21 @@ class Preset:
|
|
|
883
883
|
|
|
884
884
|
# validate excluded modules
|
|
885
885
|
for excluded_module in self.exclude_modules:
|
|
886
|
-
if not
|
|
886
|
+
if excluded_module not in self.module_loader.all_module_choices:
|
|
887
887
|
raise ValidationError(
|
|
888
888
|
get_closest_match(excluded_module, self.module_loader.all_module_choices, msg="module")
|
|
889
889
|
)
|
|
890
890
|
# validate excluded flags
|
|
891
891
|
for excluded_flag in self.exclude_flags:
|
|
892
|
-
if not
|
|
892
|
+
if excluded_flag not in self.module_loader.flag_choices:
|
|
893
893
|
raise ValidationError(get_closest_match(excluded_flag, self.module_loader.flag_choices, msg="flag"))
|
|
894
894
|
# validate required flags
|
|
895
895
|
for required_flag in self.require_flags:
|
|
896
|
-
if not
|
|
896
|
+
if required_flag not in self.module_loader.flag_choices:
|
|
897
897
|
raise ValidationError(get_closest_match(required_flag, self.module_loader.flag_choices, msg="flag"))
|
|
898
898
|
# validate flags
|
|
899
899
|
for flag in self.flags:
|
|
900
|
-
if not
|
|
900
|
+
if flag not in self.module_loader.flag_choices:
|
|
901
901
|
raise ValidationError(get_closest_match(flag, self.module_loader.flag_choices, msg="flag"))
|
|
902
902
|
|
|
903
903
|
@property
|
|
@@ -916,7 +916,7 @@ class Preset:
|
|
|
916
916
|
|
|
917
917
|
global DEFAULT_PRESETS
|
|
918
918
|
if DEFAULT_PRESETS is None:
|
|
919
|
-
presets =
|
|
919
|
+
presets = {}
|
|
920
920
|
for ext in ("yml", "yaml"):
|
|
921
921
|
for preset_path in PRESET_PATH:
|
|
922
922
|
# for every yaml file
|
|
@@ -967,7 +967,7 @@ class Preset:
|
|
|
967
967
|
header = ["Preset", "Category", "Description", "# Modules"]
|
|
968
968
|
if include_modules:
|
|
969
969
|
header.append("Modules")
|
|
970
|
-
for
|
|
970
|
+
for (loaded_preset, category, preset_path, original_file) in self.all_presets.values():
|
|
971
971
|
loaded_preset = loaded_preset.bake()
|
|
972
972
|
num_modules = f"{len(loaded_preset.scan_modules):,}"
|
|
973
973
|
row = [loaded_preset.name, category, loaded_preset.description, num_modules]
|
bbot/scanner/scanner.py
CHANGED
|
@@ -214,8 +214,8 @@ class Scanner:
|
|
|
214
214
|
)
|
|
215
215
|
|
|
216
216
|
# url file extensions
|
|
217
|
-
self.url_extension_blacklist =
|
|
218
|
-
self.url_extension_httpx_only =
|
|
217
|
+
self.url_extension_blacklist = {e.lower() for e in self.config.get("url_extension_blacklist", [])}
|
|
218
|
+
self.url_extension_httpx_only = {e.lower() for e in self.config.get("url_extension_httpx_only", [])}
|
|
219
219
|
|
|
220
220
|
# url querystring behavior
|
|
221
221
|
self.url_querystring_remove = self.config.get("url_querystring_remove", True)
|
|
@@ -338,7 +338,7 @@ class Scanner:
|
|
|
338
338
|
self.trace(f"Preset: {self.preset.to_dict(redact_secrets=True)}")
|
|
339
339
|
|
|
340
340
|
if not self.target:
|
|
341
|
-
self.warning(
|
|
341
|
+
self.warning("No scan targets specified")
|
|
342
342
|
|
|
343
343
|
# start status ticker
|
|
344
344
|
self.ticker_task = asyncio.create_task(
|
|
@@ -348,7 +348,7 @@ class Scanner:
|
|
|
348
348
|
self.status = "STARTING"
|
|
349
349
|
|
|
350
350
|
if not self.modules:
|
|
351
|
-
self.error(
|
|
351
|
+
self.error("No modules loaded")
|
|
352
352
|
self.status = "FAILED"
|
|
353
353
|
return
|
|
354
354
|
else:
|
|
@@ -451,7 +451,7 @@ class Scanner:
|
|
|
451
451
|
await m.queue_event(scan_finish_event)
|
|
452
452
|
# wait until output modules are flushed
|
|
453
453
|
while 1:
|
|
454
|
-
modules_finished = all(
|
|
454
|
+
modules_finished = all(m.finished for m in output_modules)
|
|
455
455
|
if modules_finished:
|
|
456
456
|
break
|
|
457
457
|
await asyncio.sleep(0.05)
|
|
@@ -461,7 +461,7 @@ class Scanner:
|
|
|
461
461
|
return scan_finish_event
|
|
462
462
|
|
|
463
463
|
def _start_modules(self):
|
|
464
|
-
self.verbose(
|
|
464
|
+
self.verbose("Starting module worker loops")
|
|
465
465
|
for module in self.modules.values():
|
|
466
466
|
module.start()
|
|
467
467
|
|
|
@@ -485,17 +485,17 @@ class Scanner:
|
|
|
485
485
|
Soft-failed modules are not set to an error state but are also removed if `remove_failed` is True.
|
|
486
486
|
"""
|
|
487
487
|
await self.load_modules()
|
|
488
|
-
self.verbose(
|
|
488
|
+
self.verbose("Setting up modules")
|
|
489
489
|
succeeded = []
|
|
490
490
|
hard_failed = []
|
|
491
491
|
soft_failed = []
|
|
492
492
|
|
|
493
493
|
async for task in self.helpers.as_completed([m._setup() for m in self.modules.values()]):
|
|
494
494
|
module, status, msg = await task
|
|
495
|
-
if status
|
|
495
|
+
if status is True:
|
|
496
496
|
self.debug(f"Setup succeeded for {module.name} ({msg})")
|
|
497
497
|
succeeded.append(module.name)
|
|
498
|
-
elif status
|
|
498
|
+
elif status is False:
|
|
499
499
|
self.warning(f"Setup hard-failed for {module.name}: {msg}")
|
|
500
500
|
self.modules[module.name].set_error_state()
|
|
501
501
|
hard_failed.append(module.name)
|
|
@@ -537,11 +537,11 @@ class Scanner:
|
|
|
537
537
|
"""
|
|
538
538
|
if not self._modules_loaded:
|
|
539
539
|
if not self.preset.modules:
|
|
540
|
-
self.warning(
|
|
540
|
+
self.warning("No modules to load")
|
|
541
541
|
return
|
|
542
542
|
|
|
543
543
|
if not self.preset.scan_modules:
|
|
544
|
-
self.warning(
|
|
544
|
+
self.warning("No scan modules to load")
|
|
545
545
|
|
|
546
546
|
# install module dependencies
|
|
547
547
|
succeeded, failed = await self.helpers.depsinstaller.install(*self.preset.modules)
|
|
@@ -685,7 +685,7 @@ class Scanner:
|
|
|
685
685
|
|
|
686
686
|
if modules_errored:
|
|
687
687
|
self.verbose(
|
|
688
|
-
f'{self.name}: Modules errored: {len(modules_errored):,} ({", ".join(
|
|
688
|
+
f'{self.name}: Modules errored: {len(modules_errored):,} ({", ".join(list(modules_errored))})'
|
|
689
689
|
)
|
|
690
690
|
|
|
691
691
|
num_queued_events = self.num_queued_events
|
|
@@ -722,7 +722,7 @@ class Scanner:
|
|
|
722
722
|
memory_usage = module.memory_usage
|
|
723
723
|
module_memory_usage.append((module.name, memory_usage))
|
|
724
724
|
module_memory_usage.sort(key=lambda x: x[-1], reverse=True)
|
|
725
|
-
self.debug(
|
|
725
|
+
self.debug("MODULE MEMORY USAGE:")
|
|
726
726
|
for module_name, usage in module_memory_usage:
|
|
727
727
|
self.debug(f" - {module_name}: {self.helpers.bytes_to_human(usage)}")
|
|
728
728
|
|
|
@@ -769,7 +769,7 @@ class Scanner:
|
|
|
769
769
|
# Trigger .finished() on every module and start over
|
|
770
770
|
log.info("Finishing scan")
|
|
771
771
|
for module in self.modules.values():
|
|
772
|
-
finished_event = self.make_event(
|
|
772
|
+
finished_event = self.make_event("FINISHED", "FINISHED", dummy=True, tags={module.name})
|
|
773
773
|
await module.queue_event(finished_event)
|
|
774
774
|
self.verbose("Completed finish()")
|
|
775
775
|
return True
|
|
@@ -1024,7 +1024,7 @@ class Scanner:
|
|
|
1024
1024
|
A list of DNS hostname strings generated from the scan target
|
|
1025
1025
|
"""
|
|
1026
1026
|
if self._dns_strings is None:
|
|
1027
|
-
dns_whitelist =
|
|
1027
|
+
dns_whitelist = {t.host for t in self.whitelist if t.host and isinstance(t.host, str)}
|
|
1028
1028
|
dns_whitelist = sorted(dns_whitelist, key=len)
|
|
1029
1029
|
dns_whitelist_set = set()
|
|
1030
1030
|
dns_strings = []
|
|
@@ -1121,7 +1121,7 @@ class Scanner:
|
|
|
1121
1121
|
"""
|
|
1122
1122
|
A dictionary representation of the scan including its name, ID, targets, whitelist, blacklist, and modules
|
|
1123
1123
|
"""
|
|
1124
|
-
j =
|
|
1124
|
+
j = {}
|
|
1125
1125
|
for i in ("id", "name"):
|
|
1126
1126
|
v = getattr(self, i, "")
|
|
1127
1127
|
if v:
|
|
@@ -1291,7 +1291,7 @@ class Scanner:
|
|
|
1291
1291
|
context = f"{context.__qualname__}()"
|
|
1292
1292
|
filename, lineno, funcname = self.helpers.get_traceback_details(e)
|
|
1293
1293
|
if self.helpers.in_exception_chain(e, (KeyboardInterrupt,)):
|
|
1294
|
-
log.debug(
|
|
1294
|
+
log.debug("Interrupted")
|
|
1295
1295
|
self.stop()
|
|
1296
1296
|
elif isinstance(e, BrokenPipeError):
|
|
1297
1297
|
log.debug(f"BrokenPipeError in {filename}:{lineno}:{funcname}(): {e}")
|
bbot/scanner/target.py
CHANGED
|
@@ -78,7 +78,7 @@ class BaseTarget(RadixTarget):
|
|
|
78
78
|
if args and is_event(args[0]):
|
|
79
79
|
return args[0]
|
|
80
80
|
# otherwise make a new one
|
|
81
|
-
if
|
|
81
|
+
if "tags" not in kwargs:
|
|
82
82
|
kwargs["tags"] = set()
|
|
83
83
|
kwargs["tags"].update(self.tags)
|
|
84
84
|
return make_event(*args, dummy=True, scan=self.scan, **kwargs)
|