bbot 2.5.0__py3-none-any.whl → 2.7.2.7424rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bbot/__init__.py +1 -1
- bbot/cli.py +22 -8
- bbot/core/engine.py +1 -1
- bbot/core/event/__init__.py +2 -2
- bbot/core/event/base.py +138 -110
- bbot/core/flags.py +1 -0
- bbot/core/helpers/bloom.py +6 -7
- bbot/core/helpers/command.py +5 -2
- bbot/core/helpers/depsinstaller/installer.py +78 -7
- bbot/core/helpers/dns/dns.py +0 -1
- bbot/core/helpers/dns/engine.py +0 -2
- bbot/core/helpers/files.py +2 -2
- bbot/core/helpers/git.py +17 -0
- bbot/core/helpers/helper.py +6 -5
- bbot/core/helpers/misc.py +15 -28
- bbot/core/helpers/names_generator.py +5 -0
- bbot/core/helpers/ntlm.py +0 -2
- bbot/core/helpers/regex.py +1 -1
- bbot/core/helpers/regexes.py +25 -8
- bbot/core/helpers/web/engine.py +1 -1
- bbot/core/helpers/web/web.py +2 -1
- bbot/core/modules.py +22 -60
- bbot/core/shared_deps.py +38 -0
- bbot/defaults.yml +4 -2
- bbot/modules/apkpure.py +2 -2
- bbot/modules/aspnet_bin_exposure.py +80 -0
- bbot/modules/baddns.py +1 -1
- bbot/modules/baddns_direct.py +1 -1
- bbot/modules/baddns_zone.py +1 -1
- bbot/modules/badsecrets.py +1 -1
- bbot/modules/base.py +129 -40
- bbot/modules/bucket_amazon.py +1 -1
- bbot/modules/bucket_digitalocean.py +1 -1
- bbot/modules/bucket_firebase.py +1 -1
- bbot/modules/bucket_google.py +1 -1
- bbot/modules/{bucket_azure.py → bucket_microsoft.py} +2 -2
- bbot/modules/builtwith.py +4 -2
- bbot/modules/c99.py +1 -1
- bbot/modules/dnsbimi.py +1 -4
- bbot/modules/dnsbrute.py +6 -1
- bbot/modules/dnscommonsrv.py +1 -0
- bbot/modules/dnsdumpster.py +35 -52
- bbot/modules/dnstlsrpt.py +0 -6
- bbot/modules/docker_pull.py +2 -2
- bbot/modules/emailformat.py +17 -1
- bbot/modules/ffuf.py +4 -1
- bbot/modules/ffuf_shortnames.py +6 -3
- bbot/modules/filedownload.py +8 -5
- bbot/modules/fullhunt.py +1 -1
- bbot/modules/git_clone.py +47 -22
- bbot/modules/gitdumper.py +5 -15
- bbot/modules/github_workflows.py +6 -5
- bbot/modules/gitlab_com.py +31 -0
- bbot/modules/gitlab_onprem.py +84 -0
- bbot/modules/gowitness.py +60 -30
- bbot/modules/graphql_introspection.py +145 -0
- bbot/modules/httpx.py +2 -0
- bbot/modules/hunt.py +10 -3
- bbot/modules/iis_shortnames.py +16 -7
- bbot/modules/internal/cloudcheck.py +65 -72
- bbot/modules/internal/unarchive.py +9 -3
- bbot/modules/lightfuzz/lightfuzz.py +6 -2
- bbot/modules/lightfuzz/submodules/esi.py +42 -0
- bbot/modules/{deadly/medusa.py → medusa.py} +4 -7
- bbot/modules/nuclei.py +2 -2
- bbot/modules/otx.py +9 -2
- bbot/modules/output/base.py +3 -11
- bbot/modules/paramminer_headers.py +10 -7
- bbot/modules/passivetotal.py +1 -1
- bbot/modules/portfilter.py +2 -0
- bbot/modules/portscan.py +1 -1
- bbot/modules/postman_download.py +2 -2
- bbot/modules/retirejs.py +232 -0
- bbot/modules/securitytxt.py +0 -3
- bbot/modules/sslcert.py +2 -2
- bbot/modules/subdomaincenter.py +1 -16
- bbot/modules/telerik.py +7 -2
- bbot/modules/templates/bucket.py +24 -4
- bbot/modules/templates/gitlab.py +98 -0
- bbot/modules/trufflehog.py +7 -4
- bbot/modules/wafw00f.py +2 -2
- bbot/presets/web/dotnet-audit.yml +1 -0
- bbot/presets/web/lightfuzz-heavy.yml +1 -1
- bbot/presets/web/lightfuzz-medium.yml +1 -1
- bbot/presets/web/lightfuzz-superheavy.yml +1 -1
- bbot/scanner/manager.py +44 -37
- bbot/scanner/scanner.py +17 -4
- bbot/scripts/benchmark_report.py +433 -0
- bbot/test/benchmarks/__init__.py +2 -0
- bbot/test/benchmarks/test_bloom_filter_benchmarks.py +105 -0
- bbot/test/benchmarks/test_closest_match_benchmarks.py +76 -0
- bbot/test/benchmarks/test_event_validation_benchmarks.py +438 -0
- bbot/test/benchmarks/test_excavate_benchmarks.py +291 -0
- bbot/test/benchmarks/test_ipaddress_benchmarks.py +143 -0
- bbot/test/benchmarks/test_weighted_shuffle_benchmarks.py +70 -0
- bbot/test/conftest.py +1 -1
- bbot/test/test_step_1/test_bbot_fastapi.py +2 -2
- bbot/test/test_step_1/test_events.py +22 -21
- bbot/test/test_step_1/test_helpers.py +20 -0
- bbot/test/test_step_1/test_manager_scope_accuracy.py +45 -0
- bbot/test/test_step_1/test_modules_basic.py +40 -15
- bbot/test/test_step_1/test_python_api.py +2 -2
- bbot/test/test_step_1/test_regexes.py +21 -4
- bbot/test/test_step_1/test_scan.py +7 -8
- bbot/test/test_step_1/test_web.py +46 -0
- bbot/test/test_step_2/module_tests/base.py +6 -1
- bbot/test/test_step_2/module_tests/test_module_aspnet_bin_exposure.py +73 -0
- bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py +52 -18
- bbot/test/test_step_2/module_tests/test_module_bucket_google.py +1 -1
- bbot/test/test_step_2/module_tests/{test_module_bucket_azure.py → test_module_bucket_microsoft.py} +7 -5
- bbot/test/test_step_2/module_tests/test_module_cloudcheck.py +19 -31
- bbot/test/test_step_2/module_tests/test_module_dnsbimi.py +2 -1
- bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py +3 -5
- bbot/test/test_step_2/module_tests/test_module_emailformat.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_emails.py +2 -2
- bbot/test/test_step_2/module_tests/test_module_excavate.py +64 -5
- bbot/test/test_step_2/module_tests/test_module_extractous.py +13 -1
- bbot/test/test_step_2/module_tests/test_module_github_workflows.py +10 -1
- bbot/test/test_step_2/module_tests/test_module_gitlab_com.py +66 -0
- bbot/test/test_step_2/module_tests/{test_module_gitlab.py → test_module_gitlab_onprem.py} +4 -69
- bbot/test/test_step_2/module_tests/test_module_gowitness.py +5 -5
- bbot/test/test_step_2/module_tests/test_module_graphql_introspection.py +34 -0
- bbot/test/test_step_2/module_tests/test_module_iis_shortnames.py +46 -1
- bbot/test/test_step_2/module_tests/test_module_jadx.py +9 -0
- bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +71 -3
- bbot/test/test_step_2/module_tests/test_module_nuclei.py +8 -6
- bbot/test/test_step_2/module_tests/test_module_otx.py +3 -0
- bbot/test/test_step_2/module_tests/test_module_portfilter.py +2 -0
- bbot/test/test_step_2/module_tests/test_module_retirejs.py +161 -0
- bbot/test/test_step_2/module_tests/test_module_telerik.py +1 -1
- bbot/test/test_step_2/module_tests/test_module_trufflehog.py +10 -1
- bbot/test/test_step_2/module_tests/test_module_unarchive.py +9 -0
- {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info}/METADATA +12 -9
- {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info}/RECORD +137 -124
- {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info}/WHEEL +1 -1
- {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info/licenses}/LICENSE +98 -58
- bbot/modules/binaryedge.py +0 -42
- bbot/modules/censys.py +0 -98
- bbot/modules/gitlab.py +0 -141
- bbot/modules/zoomeye.py +0 -77
- bbot/test/test_step_2/module_tests/test_module_binaryedge.py +0 -33
- bbot/test/test_step_2/module_tests/test_module_censys.py +0 -83
- bbot/test/test_step_2/module_tests/test_module_zoomeye.py +0 -35
- {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info}/entry_points.txt +0 -0
bbot/core/helpers/git.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def sanitize_git_repo(repo_folder: Path):
|
|
5
|
+
# sanitizing the git config is infeasible since there are too many different ways to do evil things
|
|
6
|
+
# instead, we move it out of .git and into the repo folder, so we don't miss any secrets etc. inside
|
|
7
|
+
config_file = repo_folder / ".git" / "config"
|
|
8
|
+
if config_file.exists():
|
|
9
|
+
config_file.rename(repo_folder / "git_config_original")
|
|
10
|
+
# move the index file
|
|
11
|
+
index_file = repo_folder / ".git" / "index"
|
|
12
|
+
if index_file.exists():
|
|
13
|
+
index_file.rename(repo_folder / "git_index_original")
|
|
14
|
+
# move the hooks folder
|
|
15
|
+
hooks_folder = repo_folder / ".git" / "hooks"
|
|
16
|
+
if hooks_folder.exists():
|
|
17
|
+
hooks_folder.rename(repo_folder / "git_hooks_original")
|
bbot/core/helpers/helper.py
CHANGED
|
@@ -89,6 +89,7 @@ class ConfigAwareHelper:
|
|
|
89
89
|
self.yara = YaraHelper(self)
|
|
90
90
|
self._dns = None
|
|
91
91
|
self._web = None
|
|
92
|
+
self._cloudcheck = None
|
|
92
93
|
self.config_aware_validators = self.validators.Validators(self)
|
|
93
94
|
self.depsinstaller = DepsInstaller(self)
|
|
94
95
|
self.word_cloud = WordCloud(self)
|
|
@@ -107,12 +108,12 @@ class ConfigAwareHelper:
|
|
|
107
108
|
return self._web
|
|
108
109
|
|
|
109
110
|
@property
|
|
110
|
-
def
|
|
111
|
-
if self.
|
|
112
|
-
from cloudcheck import
|
|
111
|
+
def cloudcheck(self):
|
|
112
|
+
if self._cloudcheck is None:
|
|
113
|
+
from cloudcheck import CloudCheck
|
|
113
114
|
|
|
114
|
-
self.
|
|
115
|
-
return self.
|
|
115
|
+
self._cloudcheck = CloudCheck()
|
|
116
|
+
return self._cloudcheck
|
|
116
117
|
|
|
117
118
|
def bloom_filter(self, size):
|
|
118
119
|
from .bloom import BloomFilter
|
bbot/core/helpers/misc.py
CHANGED
|
@@ -17,6 +17,7 @@ from unidecode import unidecode # noqa F401
|
|
|
17
17
|
from asyncio import create_task, gather, sleep, wait_for # noqa
|
|
18
18
|
from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin # noqa F401
|
|
19
19
|
|
|
20
|
+
from .git import * # noqa F401
|
|
20
21
|
from .url import * # noqa F401
|
|
21
22
|
from ... import errors
|
|
22
23
|
from . import regexes as bbot_regexes
|
|
@@ -216,26 +217,29 @@ def split_host_port(d):
|
|
|
216
217
|
host = None
|
|
217
218
|
port = None
|
|
218
219
|
scheme = None
|
|
220
|
+
|
|
221
|
+
# first, try to parse as an IP address
|
|
219
222
|
if is_ip(d):
|
|
220
223
|
return make_ip_type(d), port
|
|
221
224
|
|
|
225
|
+
# if not an IP address, try to parse as a host:port
|
|
222
226
|
match = bbot_regexes.split_host_port_regex.match(d)
|
|
223
227
|
if match is None:
|
|
224
|
-
raise ValueError(f'
|
|
228
|
+
raise ValueError(f'split_host_port() failed to parse "{d}"')
|
|
225
229
|
scheme = match.group("scheme")
|
|
226
230
|
netloc = match.group("netloc")
|
|
227
231
|
if netloc is None:
|
|
228
|
-
raise ValueError(f'
|
|
232
|
+
raise ValueError(f'split_host_port() failed to parse "{d}"')
|
|
229
233
|
|
|
230
234
|
match = bbot_regexes.extract_open_port_regex.match(netloc)
|
|
231
235
|
if match is None:
|
|
232
|
-
raise ValueError(f'
|
|
236
|
+
raise ValueError(f'split_host_port() failed to parse netloc "{netloc}" (original value: {d})')
|
|
233
237
|
|
|
234
238
|
host = match.group(2)
|
|
235
239
|
if host is None:
|
|
236
240
|
host = match.group(1)
|
|
237
241
|
if host is None:
|
|
238
|
-
raise ValueError(f'
|
|
242
|
+
raise ValueError(f'split_host_port() failed to locate host in netloc "{netloc}" (original value: {d})')
|
|
239
243
|
|
|
240
244
|
port = match.group(3)
|
|
241
245
|
if port is None and scheme is not None:
|
|
@@ -831,7 +835,9 @@ def rand_string(length=10, digits=True, numeric_only=False):
|
|
|
831
835
|
return "".join(random.choice(pool) for _ in range(length))
|
|
832
836
|
|
|
833
837
|
|
|
834
|
-
def truncate_string(s, n):
|
|
838
|
+
def truncate_string(s: str, n: int) -> str:
|
|
839
|
+
if not isinstance(s, str):
|
|
840
|
+
raise ValueError(f"Expected string, got {type(s)}")
|
|
835
841
|
if len(s) > n:
|
|
836
842
|
return s[: n - 3] + "..."
|
|
837
843
|
else:
|
|
@@ -1309,7 +1315,7 @@ def make_netloc(host, port=None):
|
|
|
1309
1315
|
return f"{host}:{port}"
|
|
1310
1316
|
|
|
1311
1317
|
|
|
1312
|
-
def which(*executables):
|
|
1318
|
+
def which(*executables, path=None):
|
|
1313
1319
|
"""Finds the full path of the first available executable from a list of executables.
|
|
1314
1320
|
|
|
1315
1321
|
Args:
|
|
@@ -1325,7 +1331,7 @@ def which(*executables):
|
|
|
1325
1331
|
import shutil
|
|
1326
1332
|
|
|
1327
1333
|
for e in executables:
|
|
1328
|
-
location = shutil.which(e)
|
|
1334
|
+
location = shutil.which(e, path=path)
|
|
1329
1335
|
if location:
|
|
1330
1336
|
return location
|
|
1331
1337
|
|
|
@@ -1642,7 +1648,7 @@ def filesize(f):
|
|
|
1642
1648
|
return 0
|
|
1643
1649
|
|
|
1644
1650
|
|
|
1645
|
-
def rm_rf(f):
|
|
1651
|
+
def rm_rf(f, ignore_errors=False):
|
|
1646
1652
|
"""Recursively delete a directory
|
|
1647
1653
|
|
|
1648
1654
|
Args:
|
|
@@ -1653,7 +1659,7 @@ def rm_rf(f):
|
|
|
1653
1659
|
"""
|
|
1654
1660
|
import shutil
|
|
1655
1661
|
|
|
1656
|
-
shutil.rmtree(f)
|
|
1662
|
+
shutil.rmtree(f, ignore_errors=ignore_errors)
|
|
1657
1663
|
|
|
1658
1664
|
|
|
1659
1665
|
def clean_old(d, keep=10, filter=lambda x: True, key=latest_mtime, reverse=True, raise_error=False):
|
|
@@ -2286,25 +2292,6 @@ def is_file(f):
|
|
|
2286
2292
|
return False
|
|
2287
2293
|
|
|
2288
2294
|
|
|
2289
|
-
def cloudcheck(ip):
|
|
2290
|
-
"""
|
|
2291
|
-
Check whether an IP address belongs to a cloud provider and returns the provider name, type, and subnet.
|
|
2292
|
-
|
|
2293
|
-
Args:
|
|
2294
|
-
ip (str): The IP address to check.
|
|
2295
|
-
|
|
2296
|
-
Returns:
|
|
2297
|
-
tuple: A tuple containing provider name (str), provider type (str), and subnet (IPv4Network).
|
|
2298
|
-
|
|
2299
|
-
Examples:
|
|
2300
|
-
>>> cloudcheck("168.62.20.37")
|
|
2301
|
-
('Azure', 'cloud', IPv4Network('168.62.0.0/19'))
|
|
2302
|
-
"""
|
|
2303
|
-
import cloudcheck as _cloudcheck
|
|
2304
|
-
|
|
2305
|
-
return _cloudcheck.check(ip)
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
2295
|
def is_async_function(f):
|
|
2309
2296
|
"""
|
|
2310
2297
|
Check if a given function is an asynchronous function.
|
|
@@ -50,6 +50,7 @@ adjectives = [
|
|
|
50
50
|
"delicious",
|
|
51
51
|
"demented",
|
|
52
52
|
"demonic",
|
|
53
|
+
"demonstrative",
|
|
53
54
|
"depraved",
|
|
54
55
|
"depressed",
|
|
55
56
|
"deranged",
|
|
@@ -172,6 +173,7 @@ adjectives = [
|
|
|
172
173
|
"pasty",
|
|
173
174
|
"peckish",
|
|
174
175
|
"pedantic",
|
|
176
|
+
"pensive",
|
|
175
177
|
"pernicious",
|
|
176
178
|
"perturbed",
|
|
177
179
|
"perverted",
|
|
@@ -201,8 +203,10 @@ adjectives = [
|
|
|
201
203
|
"reckless",
|
|
202
204
|
"reductive",
|
|
203
205
|
"ripped",
|
|
206
|
+
"ruthless",
|
|
204
207
|
"sadistic",
|
|
205
208
|
"satanic",
|
|
209
|
+
"saucy",
|
|
206
210
|
"savvy",
|
|
207
211
|
"scheming",
|
|
208
212
|
"schizophrenic",
|
|
@@ -668,6 +672,7 @@ names = [
|
|
|
668
672
|
"tracy",
|
|
669
673
|
"travis",
|
|
670
674
|
"treebeard",
|
|
675
|
+
"trent",
|
|
671
676
|
"triss",
|
|
672
677
|
"tyler",
|
|
673
678
|
"tyrell",
|
bbot/core/helpers/ntlm.py
CHANGED
|
@@ -17,11 +17,9 @@ class StrStruct(object):
|
|
|
17
17
|
self.alloc = alloc
|
|
18
18
|
self.offset = offset
|
|
19
19
|
self.raw = raw[offset : offset + length]
|
|
20
|
-
self.utf16 = False
|
|
21
20
|
|
|
22
21
|
if len(self.raw) >= 2 and self.raw[1] == "\0":
|
|
23
22
|
self.string = self.raw.decode("utf-16")
|
|
24
|
-
self.utf16 = True
|
|
25
23
|
else:
|
|
26
24
|
self.string = self.raw
|
|
27
25
|
|
bbot/core/helpers/regex.py
CHANGED
|
@@ -65,7 +65,7 @@ class RegexHelper:
|
|
|
65
65
|
|
|
66
66
|
while tasks: # While there are tasks pending
|
|
67
67
|
# Wait for the first task to complete
|
|
68
|
-
done,
|
|
68
|
+
done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
|
|
69
69
|
|
|
70
70
|
for task in done:
|
|
71
71
|
result = task.result()
|
bbot/core/helpers/regexes.py
CHANGED
|
@@ -23,13 +23,28 @@ num_regex = re.compile(r"\d+")
|
|
|
23
23
|
_ipv4_regex = r"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}"
|
|
24
24
|
ipv4_regex = re.compile(_ipv4_regex, re.I)
|
|
25
25
|
|
|
26
|
-
# IPv6
|
|
27
|
-
#
|
|
28
|
-
# (
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
|
|
32
|
-
|
|
26
|
+
# IPv6 regex breakdown:
|
|
27
|
+
#
|
|
28
|
+
# (?: # —— address body ——
|
|
29
|
+
# We have to individually account for all possible variations of: "N left hextets :: M right hextets" with N+M ≤ 8 or fully expanded 8 hextets.
|
|
30
|
+
# (?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4} # 8 hextets, no compression.
|
|
31
|
+
# | (?:[A-F0-9]{1,4}:){1,7}: # 1–7 left, then "::" (0 right).
|
|
32
|
+
# | (?:[A-F0-9]{1,4}:){1,6}:[A-F0-9]{1,4} # 1–6 left, "::", 1 right.
|
|
33
|
+
# | (?:[A-F0-9]{1,4}:){1,5}(?::[A-F0-9]{1,4}){1,2} # 1–5 left, "::", 1–2 right.
|
|
34
|
+
# | (?:[A-F0-9]{1,4}:){1,4}(?::[A-F0-9]{1,4}){1,3} # 1–4 left, "::", 1–3 right.
|
|
35
|
+
# | (?:[A-F0-9]{1,4}:){1,3}(?::[A-F0-9]{1,4}){1,4} # 1–3 left, "::", 1–4 right.
|
|
36
|
+
# | (?:[A-F0-9]{1,4}:){1,2}(?::[A-F0-9]{1,4}){1,5} # 1–2 left, "::", 1–5 right.
|
|
37
|
+
# | [A-F0-9]{1,4}:(?::[A-F0-9]{1,4}){1,6} # 1 left, "::", 1–6 right.
|
|
38
|
+
# | :(?::[A-F0-9]{1,4}){1,7} # 0 left, "::", 1–7 right.
|
|
39
|
+
# | :: # all zeros.
|
|
40
|
+
# )
|
|
41
|
+
#
|
|
42
|
+
# Notes:
|
|
43
|
+
# - Does not match IPv4-embedded forms (e.g., ::ffff:192.0.2.1).
|
|
44
|
+
# - Does not match zone IDs (e.g., %eth0).
|
|
45
|
+
# - Pure syntax check; will not validate special ranges.
|
|
46
|
+
|
|
47
|
+
_ipv6_regex = r"(?:(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}|(?:[A-F0-9]{1,4}:){1,7}:|(?:[A-F0-9]{1,4}:){1,6}:[A-F0-9]{1,4}|(?:[A-F0-9]{1,4}:){1,5}(?::[A-F0-9]{1,4}){1,2}|(?:[A-F0-9]{1,4}:){1,4}(?::[A-F0-9]{1,4}){1,3}|(?:[A-F0-9]{1,4}:){1,3}(?::[A-F0-9]{1,4}){1,4}|(?:[A-F0-9]{1,4}:){1,2}(?::[A-F0-9]{1,4}){1,5}|[A-F0-9]{1,4}:(?::[A-F0-9]{1,4}){1,6}|:(?::[A-F0-9]{1,4}){1,7}|::)"
|
|
33
48
|
ipv6_regex = re.compile(_ipv6_regex, re.I)
|
|
34
49
|
|
|
35
50
|
_ip_range_regexes = (
|
|
@@ -173,7 +188,9 @@ button_tag_regex2 = re.compile(
|
|
|
173
188
|
)
|
|
174
189
|
tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']?(?!mailto:)([^\'\"\>]+)[\"\']?[^>]*>")
|
|
175
190
|
|
|
176
|
-
|
|
191
|
+
_invalid_netloc_chars = r"\s!@#$%^&()=/?\\'\";~`<>"
|
|
192
|
+
# first char must not be a colon, even though it's a valid char for a netloc
|
|
193
|
+
valid_netloc = r"[^" + (_invalid_netloc_chars + ":") + r"]{1}[^" + _invalid_netloc_chars + "]*"
|
|
177
194
|
|
|
178
195
|
_split_host_port_regex = r"(?:(?P<scheme>[a-z0-9]{1,20})://)?(?:[^?]*@)?(?P<netloc>" + valid_netloc + ")"
|
|
179
196
|
split_host_port_regex = re.compile(_split_host_port_regex, re.I)
|
bbot/core/helpers/web/engine.py
CHANGED
|
@@ -208,7 +208,7 @@ class HTTPEngine(EngineServer):
|
|
|
208
208
|
raise
|
|
209
209
|
else:
|
|
210
210
|
log.warning(
|
|
211
|
-
f"Invalid URL (possibly due to dangerous redirect) on request to : {url}: {truncate_string(e, 200)}"
|
|
211
|
+
f"Invalid URL (possibly due to dangerous redirect) on request to : {url}: {truncate_string(str(e), 200)}"
|
|
212
212
|
)
|
|
213
213
|
log.trace(traceback.format_exc())
|
|
214
214
|
except ssl.SSLError as e:
|
bbot/core/helpers/web/web.py
CHANGED
|
@@ -267,7 +267,8 @@ class WebHelper(EngineClient):
|
|
|
267
267
|
if not path:
|
|
268
268
|
raise WordlistError(f"Invalid wordlist: {path}")
|
|
269
269
|
if "cache_hrs" not in kwargs:
|
|
270
|
-
|
|
270
|
+
# 4320 hrs = 180 days = 6 months
|
|
271
|
+
kwargs["cache_hrs"] = 4320
|
|
271
272
|
if self.parent_helper.is_url(path):
|
|
272
273
|
filename = await self.download(str(path), **kwargs)
|
|
273
274
|
if filename is None:
|
bbot/core/modules.py
CHANGED
|
@@ -56,7 +56,6 @@ class ModuleLoader:
|
|
|
56
56
|
self._shared_deps = dict(SHARED_DEPS)
|
|
57
57
|
|
|
58
58
|
self.__preloaded = {}
|
|
59
|
-
self._modules = {}
|
|
60
59
|
self._configs = {}
|
|
61
60
|
self.flag_choices = set()
|
|
62
61
|
self.all_module_choices = set()
|
|
@@ -165,8 +164,10 @@ class ModuleLoader:
|
|
|
165
164
|
if module_dir.name in ("output", "internal"):
|
|
166
165
|
module_type = str(module_dir.name)
|
|
167
166
|
|
|
167
|
+
disable_auto_module_deps = preloaded.get("disable_auto_module_deps", False)
|
|
168
|
+
|
|
168
169
|
# derive module dependencies from watched event types (only for scan modules)
|
|
169
|
-
if module_type == "scan":
|
|
170
|
+
if module_type == "scan" and not disable_auto_module_deps:
|
|
170
171
|
for event_type in preloaded["watched_events"]:
|
|
171
172
|
if event_type in self.default_module_deps:
|
|
172
173
|
deps_modules = set(preloaded.get("deps", {}).get("modules", []))
|
|
@@ -329,6 +330,7 @@ class ModuleLoader:
|
|
|
329
330
|
ansible_tasks = []
|
|
330
331
|
config = {}
|
|
331
332
|
options_desc = {}
|
|
333
|
+
disable_auto_module_deps = False
|
|
332
334
|
python_code = open(module_file).read()
|
|
333
335
|
# take a hash of the code so we can keep track of when it changes
|
|
334
336
|
module_hash = sha1(python_code).hexdigest()
|
|
@@ -353,8 +355,11 @@ class ModuleLoader:
|
|
|
353
355
|
# look for classes
|
|
354
356
|
if type(root_element) == ast.ClassDef:
|
|
355
357
|
for class_attr in root_element.body:
|
|
358
|
+
if not type(class_attr) == ast.Assign:
|
|
359
|
+
continue
|
|
360
|
+
|
|
356
361
|
# class attributes that are dictionaries
|
|
357
|
-
if type(class_attr
|
|
362
|
+
if type(class_attr.value) == ast.Dict:
|
|
358
363
|
# module options
|
|
359
364
|
if any(target.id == "options" for target in class_attr.targets):
|
|
360
365
|
config.update(ast.literal_eval(class_attr.value))
|
|
@@ -366,7 +371,7 @@ class ModuleLoader:
|
|
|
366
371
|
meta = ast.literal_eval(class_attr.value)
|
|
367
372
|
|
|
368
373
|
# class attributes that are lists
|
|
369
|
-
if type(class_attr
|
|
374
|
+
if type(class_attr.value) == ast.List:
|
|
370
375
|
# flags
|
|
371
376
|
if any(target.id == "flags" for target in class_attr.targets):
|
|
372
377
|
for flag in class_attr.value.elts:
|
|
@@ -415,6 +420,12 @@ class ModuleLoader:
|
|
|
415
420
|
if type(dep_common.value) == str:
|
|
416
421
|
deps_common.append(dep_common.value)
|
|
417
422
|
|
|
423
|
+
# class attributes that are booleans
|
|
424
|
+
if type(class_attr.value) == ast.Constant:
|
|
425
|
+
if any(target.id == "_disable_auto_module_deps" for target in class_attr.targets):
|
|
426
|
+
if type(class_attr.value.value) == bool:
|
|
427
|
+
disable_auto_module_deps = class_attr.value.value
|
|
428
|
+
|
|
418
429
|
for task in ansible_tasks:
|
|
419
430
|
if "become" not in task:
|
|
420
431
|
task["become"] = False
|
|
@@ -441,6 +452,7 @@ class ModuleLoader:
|
|
|
441
452
|
"common": deps_common,
|
|
442
453
|
},
|
|
443
454
|
"sudo": len(deps_apt) > 0,
|
|
455
|
+
"disable_auto_module_deps": disable_auto_module_deps,
|
|
444
456
|
}
|
|
445
457
|
ansible_task_list = list(ansible_tasks)
|
|
446
458
|
for dep_common in deps_common:
|
|
@@ -461,9 +473,13 @@ class ModuleLoader:
|
|
|
461
473
|
def load_modules(self, module_names):
|
|
462
474
|
modules = {}
|
|
463
475
|
for module_name in module_names:
|
|
464
|
-
|
|
476
|
+
try:
|
|
477
|
+
module = self.load_module(module_name)
|
|
478
|
+
except ModuleNotFoundError as e:
|
|
479
|
+
raise BBOTError(
|
|
480
|
+
f"Error loading module {module_name}: {e}. You may have leftover artifacts from an older version of BBOT. Try deleting/renaming your '~/.bbot' directory."
|
|
481
|
+
) from e
|
|
465
482
|
modules[module_name] = module
|
|
466
|
-
self._modules[module_name] = module
|
|
467
483
|
return modules
|
|
468
484
|
|
|
469
485
|
def load_module(self, module_name):
|
|
@@ -512,60 +528,6 @@ class ModuleLoader:
|
|
|
512
528
|
# then we have a module
|
|
513
529
|
return value
|
|
514
530
|
|
|
515
|
-
def recommend_dependencies(self, modules):
|
|
516
|
-
"""
|
|
517
|
-
Returns a dictionary containing missing dependencies and their suggested resolutions
|
|
518
|
-
|
|
519
|
-
Needs work. For this we should probably be building a dependency graph
|
|
520
|
-
"""
|
|
521
|
-
resolve_choices = {}
|
|
522
|
-
# step 1: build a dictionary containing event types and their associated modules
|
|
523
|
-
# {"IP_ADDRESS": set("masscan", "ipneighbor", ...)}
|
|
524
|
-
watched = {}
|
|
525
|
-
produced = {}
|
|
526
|
-
for modname in modules:
|
|
527
|
-
preloaded = self._preloaded.get(modname)
|
|
528
|
-
if preloaded:
|
|
529
|
-
for event_type in preloaded.get("watched_events", []):
|
|
530
|
-
self.add_or_create(watched, event_type, modname)
|
|
531
|
-
for event_type in preloaded.get("produced_events", []):
|
|
532
|
-
self.add_or_create(produced, event_type, modname)
|
|
533
|
-
watched_all = {}
|
|
534
|
-
produced_all = {}
|
|
535
|
-
for modname, preloaded in self.preloaded().items():
|
|
536
|
-
if preloaded:
|
|
537
|
-
for event_type in preloaded.get("watched_events", []):
|
|
538
|
-
self.add_or_create(watched_all, event_type, modname)
|
|
539
|
-
for event_type in preloaded.get("produced_events", []):
|
|
540
|
-
self.add_or_create(produced_all, event_type, modname)
|
|
541
|
-
|
|
542
|
-
# step 2: check to see if there are missing dependencies
|
|
543
|
-
for modname in modules:
|
|
544
|
-
preloaded = self._preloaded.get(modname)
|
|
545
|
-
module_type = preloaded.get("type", "unknown")
|
|
546
|
-
if module_type != "scan":
|
|
547
|
-
continue
|
|
548
|
-
watched_events = preloaded.get("watched_events", [])
|
|
549
|
-
missing_deps = {e: not self.check_dependency(e, modname, produced) for e in watched_events}
|
|
550
|
-
if all(missing_deps.values()):
|
|
551
|
-
for event_type in watched_events:
|
|
552
|
-
if event_type == "SCAN":
|
|
553
|
-
continue
|
|
554
|
-
choices = produced_all.get(event_type, [])
|
|
555
|
-
choices = set(choices)
|
|
556
|
-
with suppress(KeyError):
|
|
557
|
-
choices.remove(modname)
|
|
558
|
-
if event_type not in resolve_choices:
|
|
559
|
-
resolve_choices[event_type] = {}
|
|
560
|
-
deps = resolve_choices[event_type]
|
|
561
|
-
self.add_or_create(deps, "required_by", modname)
|
|
562
|
-
for c in choices:
|
|
563
|
-
choice_type = self._preloaded.get(c, {}).get("type", "unknown")
|
|
564
|
-
if choice_type == "scan":
|
|
565
|
-
self.add_or_create(deps, "recommended", c)
|
|
566
|
-
|
|
567
|
-
return resolve_choices
|
|
568
|
-
|
|
569
531
|
def check_dependency(self, event_type, modname, produced):
|
|
570
532
|
if event_type not in produced:
|
|
571
533
|
return False
|
bbot/core/shared_deps.py
CHANGED
|
@@ -108,6 +108,24 @@ DEP_CHROMIUM = [
|
|
|
108
108
|
"when": "ansible_facts['os_family'] == 'Debian'",
|
|
109
109
|
"ignore_errors": True,
|
|
110
110
|
},
|
|
111
|
+
{
|
|
112
|
+
"name": "Get latest Chromium version (Darwin x86_64)",
|
|
113
|
+
"uri": {
|
|
114
|
+
"url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2FLAST_CHANGE?alt=media",
|
|
115
|
+
"return_content": True,
|
|
116
|
+
},
|
|
117
|
+
"register": "chromium_version_darwin_x86_64",
|
|
118
|
+
"when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'",
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
"name": "Get latest Chromium version (Darwin arm64)",
|
|
122
|
+
"uri": {
|
|
123
|
+
"url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2FLAST_CHANGE?alt=media",
|
|
124
|
+
"return_content": True,
|
|
125
|
+
},
|
|
126
|
+
"register": "chromium_version_darwin_arm64",
|
|
127
|
+
"when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'",
|
|
128
|
+
},
|
|
111
129
|
{
|
|
112
130
|
"name": "Download Chromium (Debian)",
|
|
113
131
|
"unarchive": {
|
|
@@ -119,6 +137,26 @@ DEP_CHROMIUM = [
|
|
|
119
137
|
"when": "ansible_facts['os_family'] == 'Debian'",
|
|
120
138
|
"ignore_errors": True,
|
|
121
139
|
},
|
|
140
|
+
{
|
|
141
|
+
"name": "Download Chromium (Darwin x86_64)",
|
|
142
|
+
"unarchive": {
|
|
143
|
+
"src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F{{ chromium_version_darwin_x86_64.content }}%2Fchrome-mac.zip?alt=media",
|
|
144
|
+
"remote_src": True,
|
|
145
|
+
"dest": "#{BBOT_TOOLS}",
|
|
146
|
+
"creates": "#{BBOT_TOOLS}/chrome-mac",
|
|
147
|
+
},
|
|
148
|
+
"when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'",
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
"name": "Download Chromium (Darwin arm64)",
|
|
152
|
+
"unarchive": {
|
|
153
|
+
"src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2F{{ chromium_version_darwin_arm64.content }}%2Fchrome-mac.zip?alt=media",
|
|
154
|
+
"remote_src": True,
|
|
155
|
+
"dest": "#{BBOT_TOOLS}",
|
|
156
|
+
"creates": "#{BBOT_TOOLS}/chrome-mac",
|
|
157
|
+
},
|
|
158
|
+
"when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'",
|
|
159
|
+
},
|
|
122
160
|
# Because Ubuntu is a special snowflake, we have to bend over backwards to fix the chrome sandbox
|
|
123
161
|
# see https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md
|
|
124
162
|
{
|
bbot/defaults.yml
CHANGED
|
@@ -187,8 +187,10 @@ url_extension_blacklist:
|
|
|
187
187
|
- mov
|
|
188
188
|
- flv
|
|
189
189
|
- webm
|
|
190
|
-
|
|
191
|
-
|
|
190
|
+
|
|
191
|
+
# URLs with these extensions are not distributed to modules unless the module opts in via `accept_url_special = True`
|
|
192
|
+
# They are also excluded from output. If you want to see them in output, remove them from this list.
|
|
193
|
+
url_extension_special:
|
|
192
194
|
- js
|
|
193
195
|
|
|
194
196
|
# These url extensions are almost always static, so we exclude them from modules that fuzz things
|
bbot/modules/apkpure.py
CHANGED
|
@@ -6,7 +6,7 @@ from bbot.modules.base import BaseModule
|
|
|
6
6
|
class apkpure(BaseModule):
|
|
7
7
|
watched_events = ["MOBILE_APP"]
|
|
8
8
|
produced_events = ["FILESYSTEM"]
|
|
9
|
-
flags = ["passive", "safe", "code-enum"]
|
|
9
|
+
flags = ["passive", "safe", "code-enum", "download"]
|
|
10
10
|
meta = {
|
|
11
11
|
"description": "Download android applications from apkpure.com",
|
|
12
12
|
"created_date": "2024-10-11",
|
|
@@ -22,7 +22,7 @@ class apkpure(BaseModule):
|
|
|
22
22
|
if output_folder:
|
|
23
23
|
self.output_dir = Path(output_folder) / "apk_files"
|
|
24
24
|
else:
|
|
25
|
-
self.output_dir = self.
|
|
25
|
+
self.output_dir = self.scan.temp_dir / "apk_files"
|
|
26
26
|
self.helpers.mkdir(self.output_dir)
|
|
27
27
|
return await super().setup()
|
|
28
28
|
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
from bbot.modules.base import BaseModule
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class aspnet_bin_exposure(BaseModule):
|
|
5
|
+
watched_events = ["URL"]
|
|
6
|
+
produced_events = ["VULNERABILITY"]
|
|
7
|
+
flags = ["active", "safe", "web-thorough"]
|
|
8
|
+
meta = {
|
|
9
|
+
"description": "Check for ASP.NET Security Feature Bypasses (CVE-2023-36899 and CVE-2023-36560)",
|
|
10
|
+
"created_date": "2025-01-28",
|
|
11
|
+
"author": "@liquidsec",
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
in_scope_only = True
|
|
15
|
+
test_dlls = [
|
|
16
|
+
"Telerik.Web.UI.dll",
|
|
17
|
+
"Newtonsoft.Json.dll",
|
|
18
|
+
"System.Net.Http.dll",
|
|
19
|
+
"EntityFramework.dll",
|
|
20
|
+
"AjaxControlToolkit.dll",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
@staticmethod
|
|
24
|
+
def normalize_url(url):
|
|
25
|
+
return str(url.rstrip("/") + "/").lower()
|
|
26
|
+
|
|
27
|
+
def _incoming_dedup_hash(self, event):
|
|
28
|
+
return hash(self.normalize_url(event.data))
|
|
29
|
+
|
|
30
|
+
async def handle_event(self, event):
|
|
31
|
+
normalized_url = self.normalize_url(event.data)
|
|
32
|
+
for test_dll in self.test_dlls:
|
|
33
|
+
for technique in ["b/(S(X))in/###DLL_PLACEHOLDER###/(S(X))/", "(S(X))/b/(S(X))in/###DLL_PLACEHOLDER###"]:
|
|
34
|
+
test_url = f"{normalized_url}{technique.replace('###DLL_PLACEHOLDER###', test_dll)}"
|
|
35
|
+
self.debug(f"Sending test URL: [{test_url}]")
|
|
36
|
+
kwargs = {"method": "GET", "allow_redirects": False, "timeout": 10}
|
|
37
|
+
test_result = await self.helpers.request(test_url, **kwargs)
|
|
38
|
+
if test_result:
|
|
39
|
+
if test_result.status_code == 200 and (
|
|
40
|
+
"content-type" in test_result.headers
|
|
41
|
+
and "application/x-msdownload" in test_result.headers["content-type"]
|
|
42
|
+
):
|
|
43
|
+
self.debug(
|
|
44
|
+
f"Got positive result for probe with test url: [{test_url}]. Status Code: [{test_result.status_code}] Content Length: [{len(test_result.content)}]"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
if test_result.status_code == 200 and (
|
|
48
|
+
"content-type" in test_result.headers
|
|
49
|
+
and "application/x-msdownload" in test_result.headers["content-type"]
|
|
50
|
+
):
|
|
51
|
+
confirm_url = (
|
|
52
|
+
f"{normalized_url}{technique.replace('###DLL_PLACEHOLDER###', 'oopsnotarealdll.dll')}"
|
|
53
|
+
)
|
|
54
|
+
confirm_result = await self.helpers.request(confirm_url, **kwargs)
|
|
55
|
+
|
|
56
|
+
if confirm_result and (
|
|
57
|
+
confirm_result.status_code != 200
|
|
58
|
+
or not (
|
|
59
|
+
"content-type" in confirm_result.headers
|
|
60
|
+
and "application/x-msdownload" in confirm_result.headers["content-type"]
|
|
61
|
+
)
|
|
62
|
+
):
|
|
63
|
+
description = f"IIS Bin Directory DLL Exposure. Detection Url: [{test_url}]"
|
|
64
|
+
await self.emit_event(
|
|
65
|
+
{
|
|
66
|
+
"severity": "HIGH",
|
|
67
|
+
"host": str(event.host),
|
|
68
|
+
"url": normalized_url,
|
|
69
|
+
"description": description,
|
|
70
|
+
},
|
|
71
|
+
"VULNERABILITY",
|
|
72
|
+
event,
|
|
73
|
+
context="{module} detected IIS Bin Directory DLL Exposure vulnerability",
|
|
74
|
+
)
|
|
75
|
+
return True
|
|
76
|
+
|
|
77
|
+
async def filter_event(self, event):
|
|
78
|
+
if "dir" in event.tags:
|
|
79
|
+
return True
|
|
80
|
+
return False
|
bbot/modules/baddns.py
CHANGED
|
@@ -22,7 +22,7 @@ class baddns(BaseModule):
|
|
|
22
22
|
"enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only",
|
|
23
23
|
}
|
|
24
24
|
module_threads = 8
|
|
25
|
-
deps_pip = ["baddns~=1.
|
|
25
|
+
deps_pip = ["baddns~=1.12.294"]
|
|
26
26
|
|
|
27
27
|
def select_modules(self):
|
|
28
28
|
selected_submodules = []
|
bbot/modules/baddns_direct.py
CHANGED
bbot/modules/baddns_zone.py
CHANGED
|
@@ -16,7 +16,7 @@ class baddns_zone(baddns_module):
|
|
|
16
16
|
"only_high_confidence": "Do not emit low-confidence or generic detections",
|
|
17
17
|
}
|
|
18
18
|
module_threads = 8
|
|
19
|
-
deps_pip = ["baddns~=1.
|
|
19
|
+
deps_pip = ["baddns~=1.12.294"]
|
|
20
20
|
|
|
21
21
|
def set_modules(self):
|
|
22
22
|
self.enabled_submodules = ["NSEC", "zonetransfer"]
|
bbot/modules/badsecrets.py
CHANGED
|
@@ -17,7 +17,7 @@ class badsecrets(BaseModule):
|
|
|
17
17
|
options_desc = {
|
|
18
18
|
"custom_secrets": "Include custom secrets loaded from a local file",
|
|
19
19
|
}
|
|
20
|
-
deps_pip = ["badsecrets~=0.
|
|
20
|
+
deps_pip = ["badsecrets~=0.13.47"]
|
|
21
21
|
|
|
22
22
|
async def setup(self):
|
|
23
23
|
self.custom_secrets = None
|