bbot 2.5.0__py3-none-any.whl → 2.7.2.7424rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. bbot/__init__.py +1 -1
  2. bbot/cli.py +22 -8
  3. bbot/core/engine.py +1 -1
  4. bbot/core/event/__init__.py +2 -2
  5. bbot/core/event/base.py +138 -110
  6. bbot/core/flags.py +1 -0
  7. bbot/core/helpers/bloom.py +6 -7
  8. bbot/core/helpers/command.py +5 -2
  9. bbot/core/helpers/depsinstaller/installer.py +78 -7
  10. bbot/core/helpers/dns/dns.py +0 -1
  11. bbot/core/helpers/dns/engine.py +0 -2
  12. bbot/core/helpers/files.py +2 -2
  13. bbot/core/helpers/git.py +17 -0
  14. bbot/core/helpers/helper.py +6 -5
  15. bbot/core/helpers/misc.py +15 -28
  16. bbot/core/helpers/names_generator.py +5 -0
  17. bbot/core/helpers/ntlm.py +0 -2
  18. bbot/core/helpers/regex.py +1 -1
  19. bbot/core/helpers/regexes.py +25 -8
  20. bbot/core/helpers/web/engine.py +1 -1
  21. bbot/core/helpers/web/web.py +2 -1
  22. bbot/core/modules.py +22 -60
  23. bbot/core/shared_deps.py +38 -0
  24. bbot/defaults.yml +4 -2
  25. bbot/modules/apkpure.py +2 -2
  26. bbot/modules/aspnet_bin_exposure.py +80 -0
  27. bbot/modules/baddns.py +1 -1
  28. bbot/modules/baddns_direct.py +1 -1
  29. bbot/modules/baddns_zone.py +1 -1
  30. bbot/modules/badsecrets.py +1 -1
  31. bbot/modules/base.py +129 -40
  32. bbot/modules/bucket_amazon.py +1 -1
  33. bbot/modules/bucket_digitalocean.py +1 -1
  34. bbot/modules/bucket_firebase.py +1 -1
  35. bbot/modules/bucket_google.py +1 -1
  36. bbot/modules/{bucket_azure.py → bucket_microsoft.py} +2 -2
  37. bbot/modules/builtwith.py +4 -2
  38. bbot/modules/c99.py +1 -1
  39. bbot/modules/dnsbimi.py +1 -4
  40. bbot/modules/dnsbrute.py +6 -1
  41. bbot/modules/dnscommonsrv.py +1 -0
  42. bbot/modules/dnsdumpster.py +35 -52
  43. bbot/modules/dnstlsrpt.py +0 -6
  44. bbot/modules/docker_pull.py +2 -2
  45. bbot/modules/emailformat.py +17 -1
  46. bbot/modules/ffuf.py +4 -1
  47. bbot/modules/ffuf_shortnames.py +6 -3
  48. bbot/modules/filedownload.py +8 -5
  49. bbot/modules/fullhunt.py +1 -1
  50. bbot/modules/git_clone.py +47 -22
  51. bbot/modules/gitdumper.py +5 -15
  52. bbot/modules/github_workflows.py +6 -5
  53. bbot/modules/gitlab_com.py +31 -0
  54. bbot/modules/gitlab_onprem.py +84 -0
  55. bbot/modules/gowitness.py +60 -30
  56. bbot/modules/graphql_introspection.py +145 -0
  57. bbot/modules/httpx.py +2 -0
  58. bbot/modules/hunt.py +10 -3
  59. bbot/modules/iis_shortnames.py +16 -7
  60. bbot/modules/internal/cloudcheck.py +65 -72
  61. bbot/modules/internal/unarchive.py +9 -3
  62. bbot/modules/lightfuzz/lightfuzz.py +6 -2
  63. bbot/modules/lightfuzz/submodules/esi.py +42 -0
  64. bbot/modules/{deadly/medusa.py → medusa.py} +4 -7
  65. bbot/modules/nuclei.py +2 -2
  66. bbot/modules/otx.py +9 -2
  67. bbot/modules/output/base.py +3 -11
  68. bbot/modules/paramminer_headers.py +10 -7
  69. bbot/modules/passivetotal.py +1 -1
  70. bbot/modules/portfilter.py +2 -0
  71. bbot/modules/portscan.py +1 -1
  72. bbot/modules/postman_download.py +2 -2
  73. bbot/modules/retirejs.py +232 -0
  74. bbot/modules/securitytxt.py +0 -3
  75. bbot/modules/sslcert.py +2 -2
  76. bbot/modules/subdomaincenter.py +1 -16
  77. bbot/modules/telerik.py +7 -2
  78. bbot/modules/templates/bucket.py +24 -4
  79. bbot/modules/templates/gitlab.py +98 -0
  80. bbot/modules/trufflehog.py +7 -4
  81. bbot/modules/wafw00f.py +2 -2
  82. bbot/presets/web/dotnet-audit.yml +1 -0
  83. bbot/presets/web/lightfuzz-heavy.yml +1 -1
  84. bbot/presets/web/lightfuzz-medium.yml +1 -1
  85. bbot/presets/web/lightfuzz-superheavy.yml +1 -1
  86. bbot/scanner/manager.py +44 -37
  87. bbot/scanner/scanner.py +17 -4
  88. bbot/scripts/benchmark_report.py +433 -0
  89. bbot/test/benchmarks/__init__.py +2 -0
  90. bbot/test/benchmarks/test_bloom_filter_benchmarks.py +105 -0
  91. bbot/test/benchmarks/test_closest_match_benchmarks.py +76 -0
  92. bbot/test/benchmarks/test_event_validation_benchmarks.py +438 -0
  93. bbot/test/benchmarks/test_excavate_benchmarks.py +291 -0
  94. bbot/test/benchmarks/test_ipaddress_benchmarks.py +143 -0
  95. bbot/test/benchmarks/test_weighted_shuffle_benchmarks.py +70 -0
  96. bbot/test/conftest.py +1 -1
  97. bbot/test/test_step_1/test_bbot_fastapi.py +2 -2
  98. bbot/test/test_step_1/test_events.py +22 -21
  99. bbot/test/test_step_1/test_helpers.py +20 -0
  100. bbot/test/test_step_1/test_manager_scope_accuracy.py +45 -0
  101. bbot/test/test_step_1/test_modules_basic.py +40 -15
  102. bbot/test/test_step_1/test_python_api.py +2 -2
  103. bbot/test/test_step_1/test_regexes.py +21 -4
  104. bbot/test/test_step_1/test_scan.py +7 -8
  105. bbot/test/test_step_1/test_web.py +46 -0
  106. bbot/test/test_step_2/module_tests/base.py +6 -1
  107. bbot/test/test_step_2/module_tests/test_module_aspnet_bin_exposure.py +73 -0
  108. bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py +52 -18
  109. bbot/test/test_step_2/module_tests/test_module_bucket_google.py +1 -1
  110. bbot/test/test_step_2/module_tests/{test_module_bucket_azure.py → test_module_bucket_microsoft.py} +7 -5
  111. bbot/test/test_step_2/module_tests/test_module_cloudcheck.py +19 -31
  112. bbot/test/test_step_2/module_tests/test_module_dnsbimi.py +2 -1
  113. bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py +3 -5
  114. bbot/test/test_step_2/module_tests/test_module_emailformat.py +1 -1
  115. bbot/test/test_step_2/module_tests/test_module_emails.py +2 -2
  116. bbot/test/test_step_2/module_tests/test_module_excavate.py +64 -5
  117. bbot/test/test_step_2/module_tests/test_module_extractous.py +13 -1
  118. bbot/test/test_step_2/module_tests/test_module_github_workflows.py +10 -1
  119. bbot/test/test_step_2/module_tests/test_module_gitlab_com.py +66 -0
  120. bbot/test/test_step_2/module_tests/{test_module_gitlab.py → test_module_gitlab_onprem.py} +4 -69
  121. bbot/test/test_step_2/module_tests/test_module_gowitness.py +5 -5
  122. bbot/test/test_step_2/module_tests/test_module_graphql_introspection.py +34 -0
  123. bbot/test/test_step_2/module_tests/test_module_iis_shortnames.py +46 -1
  124. bbot/test/test_step_2/module_tests/test_module_jadx.py +9 -0
  125. bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +71 -3
  126. bbot/test/test_step_2/module_tests/test_module_nuclei.py +8 -6
  127. bbot/test/test_step_2/module_tests/test_module_otx.py +3 -0
  128. bbot/test/test_step_2/module_tests/test_module_portfilter.py +2 -0
  129. bbot/test/test_step_2/module_tests/test_module_retirejs.py +161 -0
  130. bbot/test/test_step_2/module_tests/test_module_telerik.py +1 -1
  131. bbot/test/test_step_2/module_tests/test_module_trufflehog.py +10 -1
  132. bbot/test/test_step_2/module_tests/test_module_unarchive.py +9 -0
  133. {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info}/METADATA +12 -9
  134. {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info}/RECORD +137 -124
  135. {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info}/WHEEL +1 -1
  136. {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info/licenses}/LICENSE +98 -58
  137. bbot/modules/binaryedge.py +0 -42
  138. bbot/modules/censys.py +0 -98
  139. bbot/modules/gitlab.py +0 -141
  140. bbot/modules/zoomeye.py +0 -77
  141. bbot/test/test_step_2/module_tests/test_module_binaryedge.py +0 -33
  142. bbot/test/test_step_2/module_tests/test_module_censys.py +0 -83
  143. bbot/test/test_step_2/module_tests/test_module_zoomeye.py +0 -35
  144. {bbot-2.5.0.dist-info → bbot-2.7.2.7424rc0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,17 @@
1
+ from pathlib import Path
2
+
3
+
4
+ def sanitize_git_repo(repo_folder: Path):
5
+ # sanitizing the git config is infeasible since there are too many different ways to do evil things
6
+ # instead, we move it out of .git and into the repo folder, so we don't miss any secrets etc. inside
7
+ config_file = repo_folder / ".git" / "config"
8
+ if config_file.exists():
9
+ config_file.rename(repo_folder / "git_config_original")
10
+ # move the index file
11
+ index_file = repo_folder / ".git" / "index"
12
+ if index_file.exists():
13
+ index_file.rename(repo_folder / "git_index_original")
14
+ # move the hooks folder
15
+ hooks_folder = repo_folder / ".git" / "hooks"
16
+ if hooks_folder.exists():
17
+ hooks_folder.rename(repo_folder / "git_hooks_original")
@@ -89,6 +89,7 @@ class ConfigAwareHelper:
89
89
  self.yara = YaraHelper(self)
90
90
  self._dns = None
91
91
  self._web = None
92
+ self._cloudcheck = None
92
93
  self.config_aware_validators = self.validators.Validators(self)
93
94
  self.depsinstaller = DepsInstaller(self)
94
95
  self.word_cloud = WordCloud(self)
@@ -107,12 +108,12 @@ class ConfigAwareHelper:
107
108
  return self._web
108
109
 
109
110
  @property
110
- def cloud(self):
111
- if self._cloud is None:
112
- from cloudcheck import cloud_providers
111
+ def cloudcheck(self):
112
+ if self._cloudcheck is None:
113
+ from cloudcheck import CloudCheck
113
114
 
114
- self._cloud = cloud_providers
115
- return self._cloud
115
+ self._cloudcheck = CloudCheck()
116
+ return self._cloudcheck
116
117
 
117
118
  def bloom_filter(self, size):
118
119
  from .bloom import BloomFilter
bbot/core/helpers/misc.py CHANGED
@@ -17,6 +17,7 @@ from unidecode import unidecode # noqa F401
17
17
  from asyncio import create_task, gather, sleep, wait_for # noqa
18
18
  from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin # noqa F401
19
19
 
20
+ from .git import * # noqa F401
20
21
  from .url import * # noqa F401
21
22
  from ... import errors
22
23
  from . import regexes as bbot_regexes
@@ -216,26 +217,29 @@ def split_host_port(d):
216
217
  host = None
217
218
  port = None
218
219
  scheme = None
220
+
221
+ # first, try to parse as an IP address
219
222
  if is_ip(d):
220
223
  return make_ip_type(d), port
221
224
 
225
+ # if not an IP address, try to parse as a host:port
222
226
  match = bbot_regexes.split_host_port_regex.match(d)
223
227
  if match is None:
224
- raise ValueError(f'split_port() failed to parse "{d}"')
228
+ raise ValueError(f'split_host_port() failed to parse "{d}"')
225
229
  scheme = match.group("scheme")
226
230
  netloc = match.group("netloc")
227
231
  if netloc is None:
228
- raise ValueError(f'split_port() failed to parse "{d}"')
232
+ raise ValueError(f'split_host_port() failed to parse "{d}"')
229
233
 
230
234
  match = bbot_regexes.extract_open_port_regex.match(netloc)
231
235
  if match is None:
232
- raise ValueError(f'split_port() failed to parse netloc "{netloc}" (original value: {d})')
236
+ raise ValueError(f'split_host_port() failed to parse netloc "{netloc}" (original value: {d})')
233
237
 
234
238
  host = match.group(2)
235
239
  if host is None:
236
240
  host = match.group(1)
237
241
  if host is None:
238
- raise ValueError(f'split_port() failed to locate host in netloc "{netloc}" (original value: {d})')
242
+ raise ValueError(f'split_host_port() failed to locate host in netloc "{netloc}" (original value: {d})')
239
243
 
240
244
  port = match.group(3)
241
245
  if port is None and scheme is not None:
@@ -831,7 +835,9 @@ def rand_string(length=10, digits=True, numeric_only=False):
831
835
  return "".join(random.choice(pool) for _ in range(length))
832
836
 
833
837
 
834
- def truncate_string(s, n):
838
+ def truncate_string(s: str, n: int) -> str:
839
+ if not isinstance(s, str):
840
+ raise ValueError(f"Expected string, got {type(s)}")
835
841
  if len(s) > n:
836
842
  return s[: n - 3] + "..."
837
843
  else:
@@ -1309,7 +1315,7 @@ def make_netloc(host, port=None):
1309
1315
  return f"{host}:{port}"
1310
1316
 
1311
1317
 
1312
- def which(*executables):
1318
+ def which(*executables, path=None):
1313
1319
  """Finds the full path of the first available executable from a list of executables.
1314
1320
 
1315
1321
  Args:
@@ -1325,7 +1331,7 @@ def which(*executables):
1325
1331
  import shutil
1326
1332
 
1327
1333
  for e in executables:
1328
- location = shutil.which(e)
1334
+ location = shutil.which(e, path=path)
1329
1335
  if location:
1330
1336
  return location
1331
1337
 
@@ -1642,7 +1648,7 @@ def filesize(f):
1642
1648
  return 0
1643
1649
 
1644
1650
 
1645
- def rm_rf(f):
1651
+ def rm_rf(f, ignore_errors=False):
1646
1652
  """Recursively delete a directory
1647
1653
 
1648
1654
  Args:
@@ -1653,7 +1659,7 @@ def rm_rf(f):
1653
1659
  """
1654
1660
  import shutil
1655
1661
 
1656
- shutil.rmtree(f)
1662
+ shutil.rmtree(f, ignore_errors=ignore_errors)
1657
1663
 
1658
1664
 
1659
1665
  def clean_old(d, keep=10, filter=lambda x: True, key=latest_mtime, reverse=True, raise_error=False):
@@ -2286,25 +2292,6 @@ def is_file(f):
2286
2292
  return False
2287
2293
 
2288
2294
 
2289
- def cloudcheck(ip):
2290
- """
2291
- Check whether an IP address belongs to a cloud provider and returns the provider name, type, and subnet.
2292
-
2293
- Args:
2294
- ip (str): The IP address to check.
2295
-
2296
- Returns:
2297
- tuple: A tuple containing provider name (str), provider type (str), and subnet (IPv4Network).
2298
-
2299
- Examples:
2300
- >>> cloudcheck("168.62.20.37")
2301
- ('Azure', 'cloud', IPv4Network('168.62.0.0/19'))
2302
- """
2303
- import cloudcheck as _cloudcheck
2304
-
2305
- return _cloudcheck.check(ip)
2306
-
2307
-
2308
2295
  def is_async_function(f):
2309
2296
  """
2310
2297
  Check if a given function is an asynchronous function.
@@ -50,6 +50,7 @@ adjectives = [
50
50
  "delicious",
51
51
  "demented",
52
52
  "demonic",
53
+ "demonstrative",
53
54
  "depraved",
54
55
  "depressed",
55
56
  "deranged",
@@ -172,6 +173,7 @@ adjectives = [
172
173
  "pasty",
173
174
  "peckish",
174
175
  "pedantic",
176
+ "pensive",
175
177
  "pernicious",
176
178
  "perturbed",
177
179
  "perverted",
@@ -201,8 +203,10 @@ adjectives = [
201
203
  "reckless",
202
204
  "reductive",
203
205
  "ripped",
206
+ "ruthless",
204
207
  "sadistic",
205
208
  "satanic",
209
+ "saucy",
206
210
  "savvy",
207
211
  "scheming",
208
212
  "schizophrenic",
@@ -668,6 +672,7 @@ names = [
668
672
  "tracy",
669
673
  "travis",
670
674
  "treebeard",
675
+ "trent",
671
676
  "triss",
672
677
  "tyler",
673
678
  "tyrell",
bbot/core/helpers/ntlm.py CHANGED
@@ -17,11 +17,9 @@ class StrStruct(object):
17
17
  self.alloc = alloc
18
18
  self.offset = offset
19
19
  self.raw = raw[offset : offset + length]
20
- self.utf16 = False
21
20
 
22
21
  if len(self.raw) >= 2 and self.raw[1] == "\0":
23
22
  self.string = self.raw.decode("utf-16")
24
- self.utf16 = True
25
23
  else:
26
24
  self.string = self.raw
27
25
 
@@ -65,7 +65,7 @@ class RegexHelper:
65
65
 
66
66
  while tasks: # While there are tasks pending
67
67
  # Wait for the first task to complete
68
- done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
68
+ done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
69
69
 
70
70
  for task in done:
71
71
  result = task.result()
@@ -23,13 +23,28 @@ num_regex = re.compile(r"\d+")
23
23
  _ipv4_regex = r"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}"
24
24
  ipv4_regex = re.compile(_ipv4_regex, re.I)
25
25
 
26
- # IPv6 is complicated, so we have accommodate alternative patterns,
27
- # :(:[A-F0-9]{1,4}){1,7} == ::1, ::ffff:1
28
- # ([A-F0-9]{1,4}:){1,7}: == 2001::, 2001:db8::, 2001:db8:0:1:2:3::
29
- # ([A-F0-9]{1,4}:){1,6}:([A-F0-9]{1,4}) == 2001::1, 2001:db8::1, 2001:db8:0:1:2:3::1
30
- # ([A-F0-9]{1,4}:){7,7}([A-F0-9]{1,4}) == 1:1:1:1:1:1:1:1, ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff
31
-
32
- _ipv6_regex = r"(:(:[A-F0-9]{1,4}){1,7}|([A-F0-9]{1,4}:){1,7}:|([A-F0-9]{1,4}:){1,6}:([A-F0-9]{1,4})|([A-F0-9]{1,4}:){7,7}([A-F0-9]{1,4}))"
26
+ # IPv6 regex breakdown:
27
+ #
28
+ # (?: # —— address body ——
29
+ # We have to individually account for all possible variations of: "N left hextets :: M right hextets" with N+M ≤ 8 or fully expanded 8 hextets.
30
+ # (?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4} # 8 hextets, no compression.
31
+ # | (?:[A-F0-9]{1,4}:){1,7}: # 1–7 left, then "::" (0 right).
32
+ # | (?:[A-F0-9]{1,4}:){1,6}:[A-F0-9]{1,4} # 1–6 left, "::", 1 right.
33
+ # | (?:[A-F0-9]{1,4}:){1,5}(?::[A-F0-9]{1,4}){1,2} # 1–5 left, "::", 1–2 right.
34
+ # | (?:[A-F0-9]{1,4}:){1,4}(?::[A-F0-9]{1,4}){1,3} # 1–4 left, "::", 1–3 right.
35
+ # | (?:[A-F0-9]{1,4}:){1,3}(?::[A-F0-9]{1,4}){1,4} # 1–3 left, "::", 1–4 right.
36
+ # | (?:[A-F0-9]{1,4}:){1,2}(?::[A-F0-9]{1,4}){1,5} # 1–2 left, "::", 1–5 right.
37
+ # | [A-F0-9]{1,4}:(?::[A-F0-9]{1,4}){1,6} # 1 left, "::", 1–6 right.
38
+ # | :(?::[A-F0-9]{1,4}){1,7} # 0 left, "::", 1–7 right.
39
+ # | :: # all zeros.
40
+ # )
41
+ #
42
+ # Notes:
43
+ # - Does not match IPv4-embedded forms (e.g., ::ffff:192.0.2.1).
44
+ # - Does not match zone IDs (e.g., %eth0).
45
+ # - Pure syntax check; will not validate special ranges.
46
+
47
+ _ipv6_regex = r"(?:(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}|(?:[A-F0-9]{1,4}:){1,7}:|(?:[A-F0-9]{1,4}:){1,6}:[A-F0-9]{1,4}|(?:[A-F0-9]{1,4}:){1,5}(?::[A-F0-9]{1,4}){1,2}|(?:[A-F0-9]{1,4}:){1,4}(?::[A-F0-9]{1,4}){1,3}|(?:[A-F0-9]{1,4}:){1,3}(?::[A-F0-9]{1,4}){1,4}|(?:[A-F0-9]{1,4}:){1,2}(?::[A-F0-9]{1,4}){1,5}|[A-F0-9]{1,4}:(?::[A-F0-9]{1,4}){1,6}|:(?::[A-F0-9]{1,4}){1,7}|::)"
33
48
  ipv6_regex = re.compile(_ipv6_regex, re.I)
34
49
 
35
50
  _ip_range_regexes = (
@@ -173,7 +188,9 @@ button_tag_regex2 = re.compile(
173
188
  )
174
189
  tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']?(?!mailto:)([^\'\"\>]+)[\"\']?[^>]*>")
175
190
 
176
- valid_netloc = r"[^\s!@#$%^&()=/?\\'\";~`<>]+"
191
+ _invalid_netloc_chars = r"\s!@#$%^&()=/?\\'\";~`<>"
192
+ # first char must not be a colon, even though it's a valid char for a netloc
193
+ valid_netloc = r"[^" + (_invalid_netloc_chars + ":") + r"]{1}[^" + _invalid_netloc_chars + "]*"
177
194
 
178
195
  _split_host_port_regex = r"(?:(?P<scheme>[a-z0-9]{1,20})://)?(?:[^?]*@)?(?P<netloc>" + valid_netloc + ")"
179
196
  split_host_port_regex = re.compile(_split_host_port_regex, re.I)
@@ -208,7 +208,7 @@ class HTTPEngine(EngineServer):
208
208
  raise
209
209
  else:
210
210
  log.warning(
211
- f"Invalid URL (possibly due to dangerous redirect) on request to : {url}: {truncate_string(e, 200)}"
211
+ f"Invalid URL (possibly due to dangerous redirect) on request to : {url}: {truncate_string(str(e), 200)}"
212
212
  )
213
213
  log.trace(traceback.format_exc())
214
214
  except ssl.SSLError as e:
@@ -267,7 +267,8 @@ class WebHelper(EngineClient):
267
267
  if not path:
268
268
  raise WordlistError(f"Invalid wordlist: {path}")
269
269
  if "cache_hrs" not in kwargs:
270
- kwargs["cache_hrs"] = 720
270
+ # 4320 hrs = 180 days = 6 months
271
+ kwargs["cache_hrs"] = 4320
271
272
  if self.parent_helper.is_url(path):
272
273
  filename = await self.download(str(path), **kwargs)
273
274
  if filename is None:
bbot/core/modules.py CHANGED
@@ -56,7 +56,6 @@ class ModuleLoader:
56
56
  self._shared_deps = dict(SHARED_DEPS)
57
57
 
58
58
  self.__preloaded = {}
59
- self._modules = {}
60
59
  self._configs = {}
61
60
  self.flag_choices = set()
62
61
  self.all_module_choices = set()
@@ -165,8 +164,10 @@ class ModuleLoader:
165
164
  if module_dir.name in ("output", "internal"):
166
165
  module_type = str(module_dir.name)
167
166
 
167
+ disable_auto_module_deps = preloaded.get("disable_auto_module_deps", False)
168
+
168
169
  # derive module dependencies from watched event types (only for scan modules)
169
- if module_type == "scan":
170
+ if module_type == "scan" and not disable_auto_module_deps:
170
171
  for event_type in preloaded["watched_events"]:
171
172
  if event_type in self.default_module_deps:
172
173
  deps_modules = set(preloaded.get("deps", {}).get("modules", []))
@@ -329,6 +330,7 @@ class ModuleLoader:
329
330
  ansible_tasks = []
330
331
  config = {}
331
332
  options_desc = {}
333
+ disable_auto_module_deps = False
332
334
  python_code = open(module_file).read()
333
335
  # take a hash of the code so we can keep track of when it changes
334
336
  module_hash = sha1(python_code).hexdigest()
@@ -353,8 +355,11 @@ class ModuleLoader:
353
355
  # look for classes
354
356
  if type(root_element) == ast.ClassDef:
355
357
  for class_attr in root_element.body:
358
+ if not type(class_attr) == ast.Assign:
359
+ continue
360
+
356
361
  # class attributes that are dictionaries
357
- if type(class_attr) == ast.Assign and type(class_attr.value) == ast.Dict:
362
+ if type(class_attr.value) == ast.Dict:
358
363
  # module options
359
364
  if any(target.id == "options" for target in class_attr.targets):
360
365
  config.update(ast.literal_eval(class_attr.value))
@@ -366,7 +371,7 @@ class ModuleLoader:
366
371
  meta = ast.literal_eval(class_attr.value)
367
372
 
368
373
  # class attributes that are lists
369
- if type(class_attr) == ast.Assign and type(class_attr.value) == ast.List:
374
+ if type(class_attr.value) == ast.List:
370
375
  # flags
371
376
  if any(target.id == "flags" for target in class_attr.targets):
372
377
  for flag in class_attr.value.elts:
@@ -415,6 +420,12 @@ class ModuleLoader:
415
420
  if type(dep_common.value) == str:
416
421
  deps_common.append(dep_common.value)
417
422
 
423
+ # class attributes that are booleans
424
+ if type(class_attr.value) == ast.Constant:
425
+ if any(target.id == "_disable_auto_module_deps" for target in class_attr.targets):
426
+ if type(class_attr.value.value) == bool:
427
+ disable_auto_module_deps = class_attr.value.value
428
+
418
429
  for task in ansible_tasks:
419
430
  if "become" not in task:
420
431
  task["become"] = False
@@ -441,6 +452,7 @@ class ModuleLoader:
441
452
  "common": deps_common,
442
453
  },
443
454
  "sudo": len(deps_apt) > 0,
455
+ "disable_auto_module_deps": disable_auto_module_deps,
444
456
  }
445
457
  ansible_task_list = list(ansible_tasks)
446
458
  for dep_common in deps_common:
@@ -461,9 +473,13 @@ class ModuleLoader:
461
473
  def load_modules(self, module_names):
462
474
  modules = {}
463
475
  for module_name in module_names:
464
- module = self.load_module(module_name)
476
+ try:
477
+ module = self.load_module(module_name)
478
+ except ModuleNotFoundError as e:
479
+ raise BBOTError(
480
+ f"Error loading module {module_name}: {e}. You may have leftover artifacts from an older version of BBOT. Try deleting/renaming your '~/.bbot' directory."
481
+ ) from e
465
482
  modules[module_name] = module
466
- self._modules[module_name] = module
467
483
  return modules
468
484
 
469
485
  def load_module(self, module_name):
@@ -512,60 +528,6 @@ class ModuleLoader:
512
528
  # then we have a module
513
529
  return value
514
530
 
515
- def recommend_dependencies(self, modules):
516
- """
517
- Returns a dictionary containing missing dependencies and their suggested resolutions
518
-
519
- Needs work. For this we should probably be building a dependency graph
520
- """
521
- resolve_choices = {}
522
- # step 1: build a dictionary containing event types and their associated modules
523
- # {"IP_ADDRESS": set("masscan", "ipneighbor", ...)}
524
- watched = {}
525
- produced = {}
526
- for modname in modules:
527
- preloaded = self._preloaded.get(modname)
528
- if preloaded:
529
- for event_type in preloaded.get("watched_events", []):
530
- self.add_or_create(watched, event_type, modname)
531
- for event_type in preloaded.get("produced_events", []):
532
- self.add_or_create(produced, event_type, modname)
533
- watched_all = {}
534
- produced_all = {}
535
- for modname, preloaded in self.preloaded().items():
536
- if preloaded:
537
- for event_type in preloaded.get("watched_events", []):
538
- self.add_or_create(watched_all, event_type, modname)
539
- for event_type in preloaded.get("produced_events", []):
540
- self.add_or_create(produced_all, event_type, modname)
541
-
542
- # step 2: check to see if there are missing dependencies
543
- for modname in modules:
544
- preloaded = self._preloaded.get(modname)
545
- module_type = preloaded.get("type", "unknown")
546
- if module_type != "scan":
547
- continue
548
- watched_events = preloaded.get("watched_events", [])
549
- missing_deps = {e: not self.check_dependency(e, modname, produced) for e in watched_events}
550
- if all(missing_deps.values()):
551
- for event_type in watched_events:
552
- if event_type == "SCAN":
553
- continue
554
- choices = produced_all.get(event_type, [])
555
- choices = set(choices)
556
- with suppress(KeyError):
557
- choices.remove(modname)
558
- if event_type not in resolve_choices:
559
- resolve_choices[event_type] = {}
560
- deps = resolve_choices[event_type]
561
- self.add_or_create(deps, "required_by", modname)
562
- for c in choices:
563
- choice_type = self._preloaded.get(c, {}).get("type", "unknown")
564
- if choice_type == "scan":
565
- self.add_or_create(deps, "recommended", c)
566
-
567
- return resolve_choices
568
-
569
531
  def check_dependency(self, event_type, modname, produced):
570
532
  if event_type not in produced:
571
533
  return False
bbot/core/shared_deps.py CHANGED
@@ -108,6 +108,24 @@ DEP_CHROMIUM = [
108
108
  "when": "ansible_facts['os_family'] == 'Debian'",
109
109
  "ignore_errors": True,
110
110
  },
111
+ {
112
+ "name": "Get latest Chromium version (Darwin x86_64)",
113
+ "uri": {
114
+ "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2FLAST_CHANGE?alt=media",
115
+ "return_content": True,
116
+ },
117
+ "register": "chromium_version_darwin_x86_64",
118
+ "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'",
119
+ },
120
+ {
121
+ "name": "Get latest Chromium version (Darwin arm64)",
122
+ "uri": {
123
+ "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2FLAST_CHANGE?alt=media",
124
+ "return_content": True,
125
+ },
126
+ "register": "chromium_version_darwin_arm64",
127
+ "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'",
128
+ },
111
129
  {
112
130
  "name": "Download Chromium (Debian)",
113
131
  "unarchive": {
@@ -119,6 +137,26 @@ DEP_CHROMIUM = [
119
137
  "when": "ansible_facts['os_family'] == 'Debian'",
120
138
  "ignore_errors": True,
121
139
  },
140
+ {
141
+ "name": "Download Chromium (Darwin x86_64)",
142
+ "unarchive": {
143
+ "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F{{ chromium_version_darwin_x86_64.content }}%2Fchrome-mac.zip?alt=media",
144
+ "remote_src": True,
145
+ "dest": "#{BBOT_TOOLS}",
146
+ "creates": "#{BBOT_TOOLS}/chrome-mac",
147
+ },
148
+ "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'",
149
+ },
150
+ {
151
+ "name": "Download Chromium (Darwin arm64)",
152
+ "unarchive": {
153
+ "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2F{{ chromium_version_darwin_arm64.content }}%2Fchrome-mac.zip?alt=media",
154
+ "remote_src": True,
155
+ "dest": "#{BBOT_TOOLS}",
156
+ "creates": "#{BBOT_TOOLS}/chrome-mac",
157
+ },
158
+ "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'",
159
+ },
122
160
  # Because Ubuntu is a special snowflake, we have to bend over backwards to fix the chrome sandbox
123
161
  # see https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md
124
162
  {
bbot/defaults.yml CHANGED
@@ -187,8 +187,10 @@ url_extension_blacklist:
187
187
  - mov
188
188
  - flv
189
189
  - webm
190
- # Distribute URLs with these extensions only to httpx (these are omitted from output)
191
- url_extension_httpx_only:
190
+
191
+ # URLs with these extensions are not distributed to modules unless the module opts in via `accept_url_special = True`
192
+ # They are also excluded from output. If you want to see them in output, remove them from this list.
193
+ url_extension_special:
192
194
  - js
193
195
 
194
196
  # These url extensions are almost always static, so we exclude them from modules that fuzz things
bbot/modules/apkpure.py CHANGED
@@ -6,7 +6,7 @@ from bbot.modules.base import BaseModule
6
6
  class apkpure(BaseModule):
7
7
  watched_events = ["MOBILE_APP"]
8
8
  produced_events = ["FILESYSTEM"]
9
- flags = ["passive", "safe", "code-enum"]
9
+ flags = ["passive", "safe", "code-enum", "download"]
10
10
  meta = {
11
11
  "description": "Download android applications from apkpure.com",
12
12
  "created_date": "2024-10-11",
@@ -22,7 +22,7 @@ class apkpure(BaseModule):
22
22
  if output_folder:
23
23
  self.output_dir = Path(output_folder) / "apk_files"
24
24
  else:
25
- self.output_dir = self.helpers.temp_dir / "apk_files"
25
+ self.output_dir = self.scan.temp_dir / "apk_files"
26
26
  self.helpers.mkdir(self.output_dir)
27
27
  return await super().setup()
28
28
 
@@ -0,0 +1,80 @@
1
+ from bbot.modules.base import BaseModule
2
+
3
+
4
+ class aspnet_bin_exposure(BaseModule):
5
+ watched_events = ["URL"]
6
+ produced_events = ["VULNERABILITY"]
7
+ flags = ["active", "safe", "web-thorough"]
8
+ meta = {
9
+ "description": "Check for ASP.NET Security Feature Bypasses (CVE-2023-36899 and CVE-2023-36560)",
10
+ "created_date": "2025-01-28",
11
+ "author": "@liquidsec",
12
+ }
13
+
14
+ in_scope_only = True
15
+ test_dlls = [
16
+ "Telerik.Web.UI.dll",
17
+ "Newtonsoft.Json.dll",
18
+ "System.Net.Http.dll",
19
+ "EntityFramework.dll",
20
+ "AjaxControlToolkit.dll",
21
+ ]
22
+
23
+ @staticmethod
24
+ def normalize_url(url):
25
+ return str(url.rstrip("/") + "/").lower()
26
+
27
+ def _incoming_dedup_hash(self, event):
28
+ return hash(self.normalize_url(event.data))
29
+
30
+ async def handle_event(self, event):
31
+ normalized_url = self.normalize_url(event.data)
32
+ for test_dll in self.test_dlls:
33
+ for technique in ["b/(S(X))in/###DLL_PLACEHOLDER###/(S(X))/", "(S(X))/b/(S(X))in/###DLL_PLACEHOLDER###"]:
34
+ test_url = f"{normalized_url}{technique.replace('###DLL_PLACEHOLDER###', test_dll)}"
35
+ self.debug(f"Sending test URL: [{test_url}]")
36
+ kwargs = {"method": "GET", "allow_redirects": False, "timeout": 10}
37
+ test_result = await self.helpers.request(test_url, **kwargs)
38
+ if test_result:
39
+ if test_result.status_code == 200 and (
40
+ "content-type" in test_result.headers
41
+ and "application/x-msdownload" in test_result.headers["content-type"]
42
+ ):
43
+ self.debug(
44
+ f"Got positive result for probe with test url: [{test_url}]. Status Code: [{test_result.status_code}] Content Length: [{len(test_result.content)}]"
45
+ )
46
+
47
+ if test_result.status_code == 200 and (
48
+ "content-type" in test_result.headers
49
+ and "application/x-msdownload" in test_result.headers["content-type"]
50
+ ):
51
+ confirm_url = (
52
+ f"{normalized_url}{technique.replace('###DLL_PLACEHOLDER###', 'oopsnotarealdll.dll')}"
53
+ )
54
+ confirm_result = await self.helpers.request(confirm_url, **kwargs)
55
+
56
+ if confirm_result and (
57
+ confirm_result.status_code != 200
58
+ or not (
59
+ "content-type" in confirm_result.headers
60
+ and "application/x-msdownload" in confirm_result.headers["content-type"]
61
+ )
62
+ ):
63
+ description = f"IIS Bin Directory DLL Exposure. Detection Url: [{test_url}]"
64
+ await self.emit_event(
65
+ {
66
+ "severity": "HIGH",
67
+ "host": str(event.host),
68
+ "url": normalized_url,
69
+ "description": description,
70
+ },
71
+ "VULNERABILITY",
72
+ event,
73
+ context="{module} detected IIS Bin Directory DLL Exposure vulnerability",
74
+ )
75
+ return True
76
+
77
+ async def filter_event(self, event):
78
+ if "dir" in event.tags:
79
+ return True
80
+ return False
bbot/modules/baddns.py CHANGED
@@ -22,7 +22,7 @@ class baddns(BaseModule):
22
22
  "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only",
23
23
  }
24
24
  module_threads = 8
25
- deps_pip = ["baddns~=1.9.130"]
25
+ deps_pip = ["baddns~=1.12.294"]
26
26
 
27
27
  def select_modules(self):
28
28
  selected_submodules = []
@@ -19,7 +19,7 @@ class baddns_direct(BaseModule):
19
19
  "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
20
20
  }
21
21
  module_threads = 8
22
- deps_pip = ["baddns~=1.9.130"]
22
+ deps_pip = ["baddns~=1.12.294"]
23
23
 
24
24
  scope_distance_modifier = 1
25
25
 
@@ -16,7 +16,7 @@ class baddns_zone(baddns_module):
16
16
  "only_high_confidence": "Do not emit low-confidence or generic detections",
17
17
  }
18
18
  module_threads = 8
19
- deps_pip = ["baddns~=1.9.130"]
19
+ deps_pip = ["baddns~=1.12.294"]
20
20
 
21
21
  def set_modules(self):
22
22
  self.enabled_submodules = ["NSEC", "zonetransfer"]
@@ -17,7 +17,7 @@ class badsecrets(BaseModule):
17
17
  options_desc = {
18
18
  "custom_secrets": "Include custom secrets loaded from a local file",
19
19
  }
20
- deps_pip = ["badsecrets~=0.9.29"]
20
+ deps_pip = ["badsecrets~=0.13.47"]
21
21
 
22
22
  async def setup(self):
23
23
  self.custom_secrets = None