bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of bbot might be problematic. Click here for more details.

Files changed (270) hide show
  1. bbot/__init__.py +1 -1
  2. bbot/cli.py +3 -7
  3. bbot/core/config/files.py +0 -1
  4. bbot/core/config/logger.py +34 -4
  5. bbot/core/core.py +21 -6
  6. bbot/core/engine.py +9 -8
  7. bbot/core/event/base.py +162 -63
  8. bbot/core/helpers/bloom.py +10 -3
  9. bbot/core/helpers/command.py +9 -8
  10. bbot/core/helpers/depsinstaller/installer.py +89 -32
  11. bbot/core/helpers/depsinstaller/sudo_askpass.py +38 -2
  12. bbot/core/helpers/diff.py +10 -10
  13. bbot/core/helpers/dns/brute.py +18 -14
  14. bbot/core/helpers/dns/dns.py +16 -15
  15. bbot/core/helpers/dns/engine.py +159 -132
  16. bbot/core/helpers/dns/helpers.py +2 -2
  17. bbot/core/helpers/dns/mock.py +26 -8
  18. bbot/core/helpers/files.py +1 -1
  19. bbot/core/helpers/helper.py +7 -4
  20. bbot/core/helpers/interactsh.py +3 -3
  21. bbot/core/helpers/libmagic.py +65 -0
  22. bbot/core/helpers/misc.py +65 -22
  23. bbot/core/helpers/names_generator.py +17 -3
  24. bbot/core/helpers/process.py +0 -20
  25. bbot/core/helpers/regex.py +1 -1
  26. bbot/core/helpers/regexes.py +12 -6
  27. bbot/core/helpers/validators.py +1 -2
  28. bbot/core/helpers/web/client.py +1 -1
  29. bbot/core/helpers/web/engine.py +18 -13
  30. bbot/core/helpers/web/web.py +25 -116
  31. bbot/core/helpers/wordcloud.py +5 -5
  32. bbot/core/modules.py +36 -27
  33. bbot/core/multiprocess.py +58 -0
  34. bbot/core/shared_deps.py +46 -3
  35. bbot/db/sql/models.py +147 -0
  36. bbot/defaults.yml +15 -10
  37. bbot/errors.py +0 -8
  38. bbot/modules/anubisdb.py +2 -2
  39. bbot/modules/apkpure.py +63 -0
  40. bbot/modules/azure_tenant.py +2 -2
  41. bbot/modules/baddns.py +35 -19
  42. bbot/modules/baddns_direct.py +92 -0
  43. bbot/modules/baddns_zone.py +3 -8
  44. bbot/modules/badsecrets.py +4 -3
  45. bbot/modules/base.py +195 -51
  46. bbot/modules/bevigil.py +7 -7
  47. bbot/modules/binaryedge.py +7 -4
  48. bbot/modules/bufferoverrun.py +47 -0
  49. bbot/modules/builtwith.py +6 -10
  50. bbot/modules/bypass403.py +5 -5
  51. bbot/modules/c99.py +10 -7
  52. bbot/modules/censys.py +9 -13
  53. bbot/modules/certspotter.py +5 -3
  54. bbot/modules/chaos.py +9 -7
  55. bbot/modules/code_repository.py +1 -0
  56. bbot/modules/columbus.py +3 -3
  57. bbot/modules/crt.py +5 -3
  58. bbot/modules/deadly/dastardly.py +1 -1
  59. bbot/modules/deadly/ffuf.py +9 -9
  60. bbot/modules/deadly/nuclei.py +3 -3
  61. bbot/modules/deadly/vhost.py +4 -3
  62. bbot/modules/dehashed.py +1 -1
  63. bbot/modules/digitorus.py +1 -1
  64. bbot/modules/dnsbimi.py +145 -0
  65. bbot/modules/dnscaa.py +3 -3
  66. bbot/modules/dnsdumpster.py +4 -4
  67. bbot/modules/dnstlsrpt.py +144 -0
  68. bbot/modules/docker_pull.py +7 -5
  69. bbot/modules/dockerhub.py +2 -2
  70. bbot/modules/dotnetnuke.py +18 -19
  71. bbot/modules/emailformat.py +1 -1
  72. bbot/modules/extractous.py +122 -0
  73. bbot/modules/filedownload.py +9 -7
  74. bbot/modules/fullhunt.py +7 -4
  75. bbot/modules/generic_ssrf.py +5 -5
  76. bbot/modules/github_codesearch.py +3 -2
  77. bbot/modules/github_org.py +4 -4
  78. bbot/modules/github_workflows.py +4 -4
  79. bbot/modules/gitlab.py +2 -5
  80. bbot/modules/google_playstore.py +93 -0
  81. bbot/modules/gowitness.py +48 -50
  82. bbot/modules/hackertarget.py +5 -3
  83. bbot/modules/host_header.py +5 -5
  84. bbot/modules/httpx.py +1 -4
  85. bbot/modules/hunterio.py +3 -9
  86. bbot/modules/iis_shortnames.py +19 -30
  87. bbot/modules/internal/cloudcheck.py +27 -12
  88. bbot/modules/internal/dnsresolve.py +250 -276
  89. bbot/modules/internal/excavate.py +100 -64
  90. bbot/modules/internal/speculate.py +42 -33
  91. bbot/modules/internetdb.py +4 -2
  92. bbot/modules/ip2location.py +3 -5
  93. bbot/modules/ipneighbor.py +1 -1
  94. bbot/modules/ipstack.py +3 -8
  95. bbot/modules/jadx.py +87 -0
  96. bbot/modules/leakix.py +11 -10
  97. bbot/modules/myssl.py +2 -2
  98. bbot/modules/newsletters.py +2 -2
  99. bbot/modules/otx.py +5 -3
  100. bbot/modules/output/asset_inventory.py +7 -7
  101. bbot/modules/output/base.py +1 -1
  102. bbot/modules/output/csv.py +1 -2
  103. bbot/modules/output/http.py +20 -14
  104. bbot/modules/output/mysql.py +51 -0
  105. bbot/modules/output/neo4j.py +7 -2
  106. bbot/modules/output/postgres.py +49 -0
  107. bbot/modules/output/slack.py +0 -1
  108. bbot/modules/output/sqlite.py +29 -0
  109. bbot/modules/output/stdout.py +2 -2
  110. bbot/modules/output/teams.py +107 -6
  111. bbot/modules/paramminer_headers.py +5 -8
  112. bbot/modules/passivetotal.py +13 -13
  113. bbot/modules/portscan.py +32 -6
  114. bbot/modules/postman.py +50 -126
  115. bbot/modules/postman_download.py +220 -0
  116. bbot/modules/rapiddns.py +3 -8
  117. bbot/modules/report/asn.py +11 -11
  118. bbot/modules/robots.py +3 -3
  119. bbot/modules/securitytrails.py +7 -10
  120. bbot/modules/securitytxt.py +128 -0
  121. bbot/modules/shodan_dns.py +7 -9
  122. bbot/modules/sitedossier.py +1 -1
  123. bbot/modules/skymem.py +2 -2
  124. bbot/modules/social.py +2 -1
  125. bbot/modules/subdomaincenter.py +1 -1
  126. bbot/modules/subdomainradar.py +160 -0
  127. bbot/modules/telerik.py +8 -8
  128. bbot/modules/templates/bucket.py +1 -1
  129. bbot/modules/templates/github.py +22 -14
  130. bbot/modules/templates/postman.py +21 -0
  131. bbot/modules/templates/shodan.py +14 -13
  132. bbot/modules/templates/sql.py +95 -0
  133. bbot/modules/templates/subdomain_enum.py +53 -17
  134. bbot/modules/templates/webhook.py +2 -4
  135. bbot/modules/trickest.py +8 -37
  136. bbot/modules/trufflehog.py +18 -3
  137. bbot/modules/url_manipulation.py +3 -3
  138. bbot/modules/urlscan.py +1 -1
  139. bbot/modules/viewdns.py +1 -1
  140. bbot/modules/virustotal.py +8 -30
  141. bbot/modules/wafw00f.py +1 -1
  142. bbot/modules/wayback.py +1 -1
  143. bbot/modules/wpscan.py +17 -11
  144. bbot/modules/zoomeye.py +11 -6
  145. bbot/presets/baddns-thorough.yml +12 -0
  146. bbot/presets/fast.yml +16 -0
  147. bbot/presets/kitchen-sink.yml +1 -0
  148. bbot/presets/spider.yml +4 -0
  149. bbot/presets/subdomain-enum.yml +7 -7
  150. bbot/scanner/manager.py +5 -16
  151. bbot/scanner/preset/args.py +44 -26
  152. bbot/scanner/preset/environ.py +7 -2
  153. bbot/scanner/preset/path.py +7 -4
  154. bbot/scanner/preset/preset.py +36 -23
  155. bbot/scanner/scanner.py +176 -63
  156. bbot/scanner/target.py +236 -434
  157. bbot/scripts/docs.py +1 -1
  158. bbot/test/bbot_fixtures.py +22 -3
  159. bbot/test/conftest.py +132 -100
  160. bbot/test/fastapi_test.py +17 -0
  161. bbot/test/owasp_mastg.apk +0 -0
  162. bbot/test/run_tests.sh +4 -4
  163. bbot/test/test.conf +2 -0
  164. bbot/test/test_step_1/test_bbot_fastapi.py +82 -0
  165. bbot/test/test_step_1/test_bloom_filter.py +2 -0
  166. bbot/test/test_step_1/test_cli.py +138 -64
  167. bbot/test/test_step_1/test_dns.py +392 -70
  168. bbot/test/test_step_1/test_engine.py +17 -17
  169. bbot/test/test_step_1/test_events.py +203 -37
  170. bbot/test/test_step_1/test_helpers.py +64 -28
  171. bbot/test/test_step_1/test_manager_deduplication.py +1 -1
  172. bbot/test/test_step_1/test_manager_scope_accuracy.py +336 -338
  173. bbot/test/test_step_1/test_modules_basic.py +69 -71
  174. bbot/test/test_step_1/test_presets.py +184 -96
  175. bbot/test/test_step_1/test_python_api.py +7 -2
  176. bbot/test/test_step_1/test_regexes.py +35 -5
  177. bbot/test/test_step_1/test_scan.py +39 -5
  178. bbot/test/test_step_1/test_scope.py +5 -4
  179. bbot/test/test_step_1/test_target.py +243 -145
  180. bbot/test/test_step_1/test_web.py +48 -10
  181. bbot/test/test_step_2/module_tests/base.py +17 -20
  182. bbot/test/test_step_2/module_tests/test_module_anubisdb.py +1 -1
  183. bbot/test/test_step_2/module_tests/test_module_apkpure.py +71 -0
  184. bbot/test/test_step_2/module_tests/test_module_asset_inventory.py +0 -1
  185. bbot/test/test_step_2/module_tests/test_module_azure_realm.py +1 -1
  186. bbot/test/test_step_2/module_tests/test_module_baddns.py +6 -6
  187. bbot/test/test_step_2/module_tests/test_module_baddns_direct.py +62 -0
  188. bbot/test/test_step_2/module_tests/test_module_bevigil.py +29 -2
  189. bbot/test/test_step_2/module_tests/test_module_binaryedge.py +4 -2
  190. bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py +2 -2
  191. bbot/test/test_step_2/module_tests/test_module_bucket_azure.py +1 -1
  192. bbot/test/test_step_2/module_tests/test_module_bufferoverrun.py +35 -0
  193. bbot/test/test_step_2/module_tests/test_module_builtwith.py +2 -2
  194. bbot/test/test_step_2/module_tests/test_module_bypass403.py +1 -1
  195. bbot/test/test_step_2/module_tests/test_module_c99.py +126 -0
  196. bbot/test/test_step_2/module_tests/test_module_censys.py +4 -1
  197. bbot/test/test_step_2/module_tests/test_module_cloudcheck.py +4 -0
  198. bbot/test/test_step_2/module_tests/test_module_code_repository.py +11 -1
  199. bbot/test/test_step_2/module_tests/test_module_columbus.py +1 -1
  200. bbot/test/test_step_2/module_tests/test_module_credshed.py +3 -3
  201. bbot/test/test_step_2/module_tests/test_module_dastardly.py +2 -1
  202. bbot/test/test_step_2/module_tests/test_module_dehashed.py +2 -2
  203. bbot/test/test_step_2/module_tests/test_module_digitorus.py +1 -1
  204. bbot/test/test_step_2/module_tests/test_module_discord.py +1 -1
  205. bbot/test/test_step_2/module_tests/test_module_dnsbimi.py +103 -0
  206. bbot/test/test_step_2/module_tests/test_module_dnsbrute.py +9 -10
  207. bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py +1 -2
  208. bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +1 -2
  209. bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py +4 -4
  210. bbot/test/test_step_2/module_tests/test_module_dnstlsrpt.py +64 -0
  211. bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +0 -8
  212. bbot/test/test_step_2/module_tests/test_module_excavate.py +17 -37
  213. bbot/test/test_step_2/module_tests/test_module_extractous.py +54 -0
  214. bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py +1 -1
  215. bbot/test/test_step_2/module_tests/test_module_filedownload.py +14 -14
  216. bbot/test/test_step_2/module_tests/test_module_git_clone.py +2 -2
  217. bbot/test/test_step_2/module_tests/test_module_github_org.py +19 -8
  218. bbot/test/test_step_2/module_tests/test_module_github_workflows.py +1 -1
  219. bbot/test/test_step_2/module_tests/test_module_gitlab.py +9 -4
  220. bbot/test/test_step_2/module_tests/test_module_google_playstore.py +83 -0
  221. bbot/test/test_step_2/module_tests/test_module_gowitness.py +4 -4
  222. bbot/test/test_step_2/module_tests/test_module_host_header.py +1 -1
  223. bbot/test/test_step_2/module_tests/test_module_http.py +4 -4
  224. bbot/test/test_step_2/module_tests/test_module_httpx.py +10 -8
  225. bbot/test/test_step_2/module_tests/test_module_hunterio.py +68 -4
  226. bbot/test/test_step_2/module_tests/test_module_jadx.py +55 -0
  227. bbot/test/test_step_2/module_tests/test_module_json.py +24 -11
  228. bbot/test/test_step_2/module_tests/test_module_leakix.py +7 -3
  229. bbot/test/test_step_2/module_tests/test_module_mysql.py +76 -0
  230. bbot/test/test_step_2/module_tests/test_module_myssl.py +1 -1
  231. bbot/test/test_step_2/module_tests/test_module_neo4j.py +1 -1
  232. bbot/test/test_step_2/module_tests/test_module_newsletters.py +6 -6
  233. bbot/test/test_step_2/module_tests/test_module_ntlm.py +7 -7
  234. bbot/test/test_step_2/module_tests/test_module_oauth.py +1 -1
  235. bbot/test/test_step_2/module_tests/test_module_otx.py +1 -1
  236. bbot/test/test_step_2/module_tests/test_module_paramminer_cookies.py +1 -2
  237. bbot/test/test_step_2/module_tests/test_module_paramminer_getparams.py +0 -6
  238. bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py +2 -9
  239. bbot/test/test_step_2/module_tests/test_module_passivetotal.py +3 -1
  240. bbot/test/test_step_2/module_tests/test_module_portscan.py +9 -8
  241. bbot/test/test_step_2/module_tests/test_module_postgres.py +74 -0
  242. bbot/test/test_step_2/module_tests/test_module_postman.py +84 -253
  243. bbot/test/test_step_2/module_tests/test_module_postman_download.py +439 -0
  244. bbot/test/test_step_2/module_tests/test_module_rapiddns.py +93 -1
  245. bbot/test/test_step_2/module_tests/test_module_securitytxt.py +50 -0
  246. bbot/test/test_step_2/module_tests/test_module_shodan_dns.py +20 -1
  247. bbot/test/test_step_2/module_tests/test_module_sitedossier.py +2 -2
  248. bbot/test/test_step_2/module_tests/test_module_smuggler.py +1 -1
  249. bbot/test/test_step_2/module_tests/test_module_social.py +11 -1
  250. bbot/test/test_step_2/module_tests/test_module_speculate.py +2 -6
  251. bbot/test/test_step_2/module_tests/test_module_splunk.py +4 -4
  252. bbot/test/test_step_2/module_tests/test_module_sqlite.py +18 -0
  253. bbot/test/test_step_2/module_tests/test_module_sslcert.py +1 -1
  254. bbot/test/test_step_2/module_tests/test_module_stdout.py +5 -3
  255. bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py +1 -1
  256. bbot/test/test_step_2/module_tests/test_module_subdomainradar.py +208 -0
  257. bbot/test/test_step_2/module_tests/test_module_subdomains.py +1 -1
  258. bbot/test/test_step_2/module_tests/test_module_teams.py +8 -6
  259. bbot/test/test_step_2/module_tests/test_module_telerik.py +1 -1
  260. bbot/test/test_step_2/module_tests/test_module_trufflehog.py +317 -11
  261. bbot/test/test_step_2/module_tests/test_module_wayback.py +1 -1
  262. bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py +135 -0
  263. {bbot-2.0.1.4654rc0.dist-info → bbot-2.3.0.5397rc0.dist-info}/METADATA +48 -18
  264. bbot-2.3.0.5397rc0.dist-info/RECORD +421 -0
  265. {bbot-2.0.1.4654rc0.dist-info → bbot-2.3.0.5397rc0.dist-info}/WHEEL +1 -1
  266. bbot/modules/unstructured.py +0 -163
  267. bbot/test/test_step_2/module_tests/test_module_unstructured.py +0 -102
  268. bbot-2.0.1.4654rc0.dist-info/RECORD +0 -385
  269. {bbot-2.0.1.4654rc0.dist-info → bbot-2.3.0.5397rc0.dist-info}/LICENSE +0 -0
  270. {bbot-2.0.1.4654rc0.dist-info → bbot-2.3.0.5397rc0.dist-info}/entry_points.txt +0 -0
@@ -38,7 +38,7 @@ class asn(BaseReportModule):
38
38
 
39
39
  async def handle_event(self, event):
40
40
  host = event.host
41
- if self.cache_get(host) == False:
41
+ if self.cache_get(host) is False:
42
42
  asns, source = await self.get_asn(host)
43
43
  if not asns:
44
44
  self.cache_put(self.unknown_asn)
@@ -96,7 +96,7 @@ class asn(BaseReportModule):
96
96
  for p in self.helpers.ip_network_parents(ip):
97
97
  try:
98
98
  self.asn_counts[p] += 1
99
- if ret == False:
99
+ if ret is False:
100
100
  ret = p
101
101
  except KeyError:
102
102
  continue
@@ -112,7 +112,7 @@ class asn(BaseReportModule):
112
112
  for i, source in enumerate(list(self.sources)):
113
113
  get_asn_fn = getattr(self, f"get_asn_{source}")
114
114
  res = await get_asn_fn(ip)
115
- if res == False:
115
+ if res is False:
116
116
  # demote the current source to lowest priority since it just failed
117
117
  self.sources.append(self.sources.pop(i))
118
118
  self.verbose(f"Failed to contact {source}, retrying")
@@ -125,7 +125,7 @@ class asn(BaseReportModule):
125
125
  url = f"https://stat.ripe.net/data/network-info/data.json?resource={ip}"
126
126
  response = await self.get_url(url, "ASN")
127
127
  asns = []
128
- if response == False:
128
+ if response is False:
129
129
  return False
130
130
  data = response.get("data", {})
131
131
  if not data:
@@ -138,7 +138,7 @@ class asn(BaseReportModule):
138
138
  asn_numbers = []
139
139
  for number in asn_numbers:
140
140
  asn = await self.get_asn_metadata_ripe(number)
141
- if asn == False:
141
+ if asn is False:
142
142
  return False
143
143
  asn["subnet"] = prefix
144
144
  asns.append(asn)
@@ -155,7 +155,7 @@ class asn(BaseReportModule):
155
155
  }
156
156
  url = f"https://stat.ripe.net/data/whois/data.json?resource={asn_number}"
157
157
  response = await self.get_url(url, "ASN Metadata", cache=True)
158
- if response == False:
158
+ if response is False:
159
159
  return False
160
160
  data = response.get("data", {})
161
161
  if not data:
@@ -187,7 +187,7 @@ class asn(BaseReportModule):
187
187
  data = await self.get_url(url, "ASN")
188
188
  asns = []
189
189
  asns_tried = set()
190
- if data == False:
190
+ if data is False:
191
191
  return False
192
192
  data = data.get("data", {})
193
193
  prefixes = data.get("prefixes", [])
@@ -201,13 +201,13 @@ class asn(BaseReportModule):
201
201
  description = details.get("description") or prefix.get("description") or ""
202
202
  country = details.get("country_code") or prefix.get("country_code") or ""
203
203
  emails = []
204
- if not asn in asns_tried:
204
+ if asn not in asns_tried:
205
205
  emails = await self.get_emails_bgpview(asn)
206
- if emails == False:
206
+ if emails is False:
207
207
  return False
208
208
  asns_tried.add(asn)
209
209
  asns.append(
210
- dict(asn=asn, subnet=subnet, name=name, description=description, country=country, emails=emails)
210
+ {"asn": asn, "subnet": subnet, "name": name, "description": description, "country": country, "emails": emails}
211
211
  )
212
212
  if not asns:
213
213
  self.debug(f'No results for "{ip}"')
@@ -217,7 +217,7 @@ class asn(BaseReportModule):
217
217
  contacts = []
218
218
  url = f"https://api.bgpview.io/asn/{asn}"
219
219
  data = await self.get_url(url, "ASN metadata", cache=True)
220
- if data == False:
220
+ if data is False:
221
221
  return False
222
222
  data = data.get("data", {})
223
223
  if not data:
bbot/modules/robots.py CHANGED
@@ -33,14 +33,14 @@ class robots(BaseModule):
33
33
  for l in lines:
34
34
  if len(l) > 0:
35
35
  split_l = l.split(": ")
36
- if (split_l[0].lower() == "allow" and self.config.get("include_allow") == True) or (
37
- split_l[0].lower() == "disallow" and self.config.get("include_disallow") == True
36
+ if (split_l[0].lower() == "allow" and self.config.get("include_allow") is True) or (
37
+ split_l[0].lower() == "disallow" and self.config.get("include_disallow") is True
38
38
  ):
39
39
  unverified_url = f"{host}{split_l[1].lstrip('/')}".replace(
40
40
  "*", self.helpers.rand_string(4)
41
41
  )
42
42
 
43
- elif split_l[0].lower() == "sitemap" and self.config.get("include_sitemap") == True:
43
+ elif split_l[0].lower() == "sitemap" and self.config.get("include_sitemap") is True:
44
44
  unverified_url = split_l[1]
45
45
  else:
46
46
  continue
@@ -15,24 +15,21 @@ class securitytrails(subdomain_enum_apikey):
15
15
  options_desc = {"api_key": "SecurityTrails API key"}
16
16
 
17
17
  base_url = "https://api.securitytrails.com/v1"
18
+ ping_url = f"{base_url}/ping?apikey={{api_key}}"
18
19
 
19
20
  async def setup(self):
20
21
  self.limit = 100
21
22
  return await super().setup()
22
23
 
23
- async def ping(self):
24
- url = f"{self.base_url}/ping?apikey={self.api_key}"
25
- r = await self.request_with_fail_count(url)
26
- resp_content = getattr(r, "text", "")
27
- assert getattr(r, "status_code", 0) == 200, resp_content
28
-
29
24
  async def request_url(self, query):
30
- url = f"{self.base_url}/domain/{query}/subdomains?apikey={self.api_key}"
31
- response = await self.request_with_fail_count(url)
25
+ url = f"{self.base_url}/domain/{query}/subdomains?apikey={{api_key}}"
26
+ response = await self.api_request(url)
32
27
  return response
33
28
 
34
- def parse_results(self, r, query):
29
+ async def parse_results(self, r, query):
30
+ results = set()
35
31
  j = r.json()
36
32
  if isinstance(j, dict):
37
33
  for host in j.get("subdomains", []):
38
- yield f"{host}.{query}"
34
+ results.add(f"{host}.{query}")
35
+ return results
@@ -0,0 +1,128 @@
1
+ # securitytxt.py
2
+ #
3
+ # Checks for/parses https://target.domain/.well-known/security.txt
4
+ #
5
+ # Refer to: https://securitytxt.org/
6
+ #
7
+ # security.txt may contain email addresses and URL's, and possibly IP addresses.
8
+ #
9
+ # Example security.txt:
10
+ #
11
+ # Contact: mailto:security.reports@example.com
12
+ # Expires: 2028-05-31T14:00:00.000Z
13
+ # Encryption: https://example.com/security.pgp
14
+ # Preferred-Languages: en, es
15
+ # Canonical: https://example.com/.well-known/security.txt
16
+ # Canonical: https://www.example.com/.well-known/security.txt
17
+ # Policy: https://example.com/security-policy.html
18
+ # Hiring: https://example.com/jobs.html
19
+ #
20
+ # Example security.txt with PGP signature:
21
+ #
22
+ # -----BEGIN PGP SIGNED MESSAGE-----
23
+ # Hash: SHA512
24
+ #
25
+ # Contact: https://vdp.example.com
26
+ # Expires: 2025-01-01T00:00:00.000Z
27
+ # Preferred-Languages: fr, en
28
+ # Canonical: https://example.com/.well-known/security.txt
29
+ # Policy: https://example.com/cert
30
+ # Hiring: https://www.careers.example.com
31
+ # -----BEGIN PGP SIGNATURE-----
32
+ #
33
+ # iQIzBAEBCgAdFiEELC1a63jHPhyV60KPsvWy9dDkrigFAmJBypcACgkQsvWy9dDk
34
+ # rijXHQ//Qya3hUSy5PYW+fI3eFP1+ak6gYq3Cbzkf57cqiBhxGetIGIGNJ6mxgjS
35
+ # KAuvXLMUWgZD73r//fjZ5v1lpuWmpt54+ecat4DgcVCvFKYpaH+KBlay8SX7XtQH
36
+ # 9T2NXMcez353TMR3EUOdLwdBzGZprf0Ekg9EzaHKMk0k+A4D9CnSb8Y6BKDPC7wr
37
+ # eadwDIR9ESo0va4sjjcllCG9MF5hqK25SfsKriCSEAMhse2FToEBbw8ImkPKowMN
38
+ # whJ4MIVlBxybu6XoIyk3n7HRRduijywy7uV80pAkhk/hL6wiW3M956FiahfRI6ad
39
+ # +Gky/Ri5TjwAE/x5DhUH8O2toPsn71DeIE4geKfz5d/v41K0yncdrHjzbj0CAHu3
40
+ # wVWLKnEp8RVqTlOR8jU0HqQUQy8iZk4LY91ROv+QjG/jUTWlwun8Ljh+YUeJTMRp
41
+ # MGftCdCrrYjIy5aEQqWztt+dXKac/9e1plq3yyfuW1L+wG3zS7X+NpIJgygMvEwT
42
+ # L3dqfQf63sjk8kWIZMVnicHBlc6BiLqUn020l+pkIOr4MuuJmIlByhlnfqH7YM8k
43
+ # VShwDx7rs4Hj08C7NVCYIySaM2jM4eNKGt9V5k1F1sklCVfYaT8OqOhJrzhcisOC
44
+ # YcQDhjt/iZTR8SzrHO7kFZbaskIp2P7JMaPax2fov15AnNHQQq8=
45
+ # =8vfR
46
+ # -----END PGP SIGNATURE-----
47
+
48
+ from bbot.modules.base import BaseModule
49
+
50
+ import re
51
+
52
+ from bbot.core.helpers.regexes import email_regex, url_regexes
53
+
54
+ _securitytxt_regex = r"^(?P<k>\w+): *(?P<v>.*)$"
55
+ securitytxt_regex = re.compile(_securitytxt_regex, re.I | re.M)
56
+
57
+
58
+ class securitytxt(BaseModule):
59
+ watched_events = ["DNS_NAME"]
60
+ produced_events = ["EMAIL_ADDRESS", "URL_UNVERIFIED"]
61
+ flags = ["subdomain-enum", "cloud-enum", "active", "web-basic", "safe"]
62
+ meta = {
63
+ "description": "Check for security.txt content",
64
+ "author": "@colin-stubbs",
65
+ "created_date": "2024-05-26",
66
+ }
67
+ options = {
68
+ "emails": True,
69
+ "urls": True,
70
+ }
71
+ options_desc = {
72
+ "emails": "emit EMAIL_ADDRESS events",
73
+ "urls": "emit URL_UNVERIFIED events",
74
+ }
75
+
76
+ async def setup(self):
77
+ self._emails = self.config.get("emails", True)
78
+ self._urls = self.config.get("urls", True)
79
+ return await super().setup()
80
+
81
+ def _incoming_dedup_hash(self, event):
82
+ # dedupe by parent
83
+ parent_domain = self.helpers.parent_domain(event.data)
84
+ return hash(parent_domain), "already processed parent domain"
85
+
86
+ async def filter_event(self, event):
87
+ if "_wildcard" in str(event.host).split("."):
88
+ return False, "event is wildcard"
89
+ return True
90
+
91
+ async def handle_event(self, event):
92
+ tags = ["securitytxt-policy"]
93
+ url = f"https://{event.host}/.well-known/security.txt"
94
+
95
+ r = await self.helpers.request(url, method="GET")
96
+
97
+ if r is None or r.status_code != 200:
98
+ # it doesn't look like we got a valid response...
99
+ return
100
+
101
+ try:
102
+ s = r.text
103
+ except Exception:
104
+ s = ""
105
+
106
+ # avoid parsing the response unless it looks, at a very basic level, like an actual security.txt
107
+ s_lower = s.lower()
108
+ if "contact: " in s_lower or "expires: " in s_lower:
109
+ for securitytxt_match in securitytxt_regex.finditer(s):
110
+ v = securitytxt_match.group("v")
111
+
112
+ for match in email_regex.finditer(v):
113
+ start, end = match.span()
114
+ email = v[start:end]
115
+
116
+ if self._emails:
117
+ await self.emit_event(email, "EMAIL_ADDRESS", parent=event, tags=tags)
118
+
119
+ for url_regex in url_regexes:
120
+ for match in url_regex.finditer(v):
121
+ start, end = match.span()
122
+ found_url = v[start:end]
123
+
124
+ if found_url != url and self._urls is True:
125
+ await self.emit_event(found_url, "URL_UNVERIFIED", parent=event, tags=tags)
126
+
127
+
128
+ # EOF
@@ -16,13 +16,11 @@ class shodan_dns(shodan):
16
16
 
17
17
  base_url = "https://api.shodan.io"
18
18
 
19
- async def request_url(self, query):
20
- url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={self.api_key}"
21
- response = await self.request_with_fail_count(url)
22
- return response
19
+ async def handle_event(self, event):
20
+ await self.handle_event_paginated(event)
23
21
 
24
- def parse_results(self, r, query):
25
- json = r.json()
26
- if json:
27
- for hostname in json.get("subdomains", []):
28
- yield f"{hostname}.{query}"
22
+ def make_url(self, query):
23
+ return f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}"
24
+
25
+ async def parse_results(self, json, query):
26
+ return [f"{sub}.{query}" for sub in json.get("subdomains", [])]
@@ -52,5 +52,5 @@ class sitedossier(subdomain_enum):
52
52
  results.add(hostname)
53
53
  yield hostname
54
54
  if '<a href="/parentdomain/' not in response.text:
55
- self.debug(f"Next page not found")
55
+ self.debug("Next page not found")
56
56
  break
bbot/modules/skymem.py CHANGED
@@ -24,7 +24,7 @@ class skymem(emailformat):
24
24
  _, query = self.helpers.split_domain(event.data)
25
25
  # get first page
26
26
  url = f"{self.base_url}/srch?q={self.helpers.quote(query)}"
27
- r = await self.request_with_fail_count(url)
27
+ r = await self.api_request(url)
28
28
  if not r:
29
29
  return
30
30
  responses = [r]
@@ -34,7 +34,7 @@ class skymem(emailformat):
34
34
  if domain_ids:
35
35
  domain_id = domain_ids[0]
36
36
  for page in range(2, 22):
37
- r2 = await self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}")
37
+ r2 = await self.api_request(f"{self.base_url}/domain/{domain_id}?p={page}")
38
38
  if not r2:
39
39
  continue
40
40
  responses.append(r2)
bbot/modules/social.py CHANGED
@@ -25,6 +25,7 @@ class social(BaseModule):
25
25
  "discord": (r"discord.gg/([a-zA-Z0-9_-]+)", True),
26
26
  "docker": (r"hub.docker.com/[ru]/([a-zA-Z0-9_-]+)", False),
27
27
  "huggingface": (r"huggingface.co/([a-zA-Z0-9_-]+)", False),
28
+ "postman": (r"www.postman.com/([a-zA-Z0-9_-]+)", False),
28
29
  }
29
30
 
30
31
  scope_distance_modifier = 1
@@ -44,7 +45,7 @@ class social(BaseModule):
44
45
  url = f"https://{url}"
45
46
  event_data = {"platform": platform, "url": url, "profile_name": profile_name}
46
47
  # only emit if the same event isn't already in the parent chain
47
- if not any([e.type == "SOCIAL" and e.data == event_data for e in event.get_parents()]):
48
+ if not any(e.type == "SOCIAL" and e.data == event_data for e in event.get_parents()):
48
49
  social_event = self.make_event(
49
50
  event_data,
50
51
  "SOCIAL",
@@ -33,7 +33,7 @@ class subdomaincenter(subdomain_enum):
33
33
  break
34
34
  return response
35
35
 
36
- def parse_results(self, r, query):
36
+ async def parse_results(self, r, query):
37
37
  results = set()
38
38
  json = r.json()
39
39
  if json and isinstance(json, list):
@@ -0,0 +1,160 @@
1
+ import time
2
+ import asyncio
3
+
4
+ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
5
+
6
+
7
+ class SubdomainRadar(subdomain_enum_apikey):
8
+ watched_events = ["DNS_NAME"]
9
+ produced_events = ["DNS_NAME"]
10
+ flags = ["subdomain-enum", "passive", "safe"]
11
+ meta = {
12
+ "description": "Query the Subdomain API for subdomains",
13
+ "created_date": "2022-07-08",
14
+ "author": "@TheTechromancer",
15
+ "auth_required": True,
16
+ }
17
+ options = {"api_key": "", "group": "fast", "timeout": 120}
18
+ options_desc = {
19
+ "api_key": "SubDomainRadar.io API key",
20
+ "group": "The enumeration group to use. Choose from fast, medium, deep",
21
+ "timeout": "Timeout in seconds",
22
+ }
23
+
24
+ base_url = "https://api.subdomainradar.io"
25
+ ping_url = f"{base_url}/profile"
26
+ group_choices = ("fast", "medium", "deep")
27
+
28
+ # set this really high so the poll loop finishes as soon as possible
29
+ _qsize = 9999999
30
+
31
+ async def setup(self):
32
+ self.group = self.config.get("group", "fast").strip().lower()
33
+ self.timeout = self.config.get("timeout", 120)
34
+ if self.group not in self.group_choices:
35
+ return False, f'Invalid group: "{self.group}", please choose from {",".join(self.group_choices)}'
36
+ success, reason = await self.require_api_key()
37
+ if not success:
38
+ return success, reason
39
+ # convert groups to enumerators
40
+ enumerators = {}
41
+ response = await self.api_request(f"{self.base_url}/enumerators/groups")
42
+ status_code = getattr(response, "status_code", 0)
43
+ if status_code != 200:
44
+ return False, f"Failed to get enumerators: (HTTP status code: {status_code})"
45
+ else:
46
+ try:
47
+ j = response.json()
48
+ except Exception:
49
+ return False, "Failed to get enumerators: failed to parse response as JSON"
50
+ for group in j:
51
+ group_name = group.get("name", "").strip().lower()
52
+ if group_name:
53
+ group_enumerators = []
54
+ for enumerator in group.get("enumerators", []):
55
+ enumerator_name = enumerator.get("display_name", "")
56
+ if enumerator_name:
57
+ group_enumerators.append(enumerator_name)
58
+ if group_enumerators:
59
+ enumerators[group_name] = group_enumerators
60
+
61
+ self.enumerators = enumerators.get(self.group, [])
62
+ if not self.enumerators:
63
+ return False, f'No enumerators found for group: "{self.group}" ({self.enumerators})'
64
+
65
+ self.enum_tasks = {}
66
+ self.poll_task = asyncio.create_task(self.task_poll_loop())
67
+
68
+ return True
69
+
70
+ def prepare_api_request(self, url, kwargs):
71
+ if self.api_key:
72
+ kwargs["headers"] = {"Authorization": f"Bearer {self.api_key}"}
73
+ return url, kwargs
74
+
75
+ async def handle_event(self, event):
76
+ query = self.make_query(event)
77
+ # start enumeration task
78
+ url = f"{self.base_url}/enumerate"
79
+ response = await self.api_request(
80
+ url, method="POST", json={"domains": [query], "enumerators": self.enumerators}
81
+ )
82
+ try:
83
+ j = response.json()
84
+ except Exception:
85
+ self.warning(f"Failed to parse response as JSON: {getattr(response, 'text', '')}")
86
+ return
87
+ task_id = j.get("tasks", {}).get(query, "")
88
+ if not task_id:
89
+ self.warning(f"Failed to start enumeration for {query}")
90
+ return
91
+ self.enum_tasks[query] = (task_id, time.time(), event)
92
+ self.debug(f"Started enumeration task for {query}; task id: {task_id}")
93
+
94
+ async def task_poll_loop(self):
95
+ # async with self._task_counter.count(f"{self.name}.task_poll_loop()"):
96
+ while 1:
97
+ for query, (task_id, start_time, event) in list(self.enum_tasks.items()):
98
+ url = f"{self.base_url}/tasks/{task_id}"
99
+ response = await self.api_request(url)
100
+ if getattr(response, "status_code", 0) == 200:
101
+ finished = await self.parse_response(response, query, event)
102
+ if finished:
103
+ self.enum_tasks.pop(query)
104
+ continue
105
+ # if scan is finishing, consider timeout
106
+ if self.scan.status == "FINISHING":
107
+ if start_time + self.timeout < time.time():
108
+ self.enum_tasks.pop(query)
109
+ self.info(f"Enumeration task for {query} timed out")
110
+
111
+ if self.scan.status == "FINISHING" and not self.enum_tasks:
112
+ break
113
+ await self.helpers.sleep(5)
114
+
115
+ async def parse_response(self, response, query, event):
116
+ j = response.json()
117
+ status = j.get("status", "")
118
+ if status.lower() == "completed":
119
+ for subdomain in j.get("subdomains", []):
120
+ hostname = subdomain.get("subdomain", "")
121
+ if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
122
+ await self.emit_event(
123
+ hostname,
124
+ "DNS_NAME",
125
+ event,
126
+ abort_if=self.abort_if,
127
+ context=f'{{module}} searched SubDomainRadar.io API for "{query}" and found {{event.type}}: {{event.data}}',
128
+ )
129
+ return True
130
+ return False
131
+
132
+ async def finish(self):
133
+ start_time = time.time()
134
+ while self.enum_tasks and not self.poll_task.done():
135
+ elapsed_time = time.time() - start_time
136
+ if elapsed_time >= self.timeout:
137
+ self.warning(f"Timed out waiting for the following tasks to finish: {self.enum_tasks}")
138
+ for query, (task_id, _, _) in list(self.enum_tasks.items()):
139
+ url = f"{self.base_url}/tasks/{task_id}"
140
+ self.warning(f" - {query} ({url})")
141
+ break
142
+
143
+ self.verbose(
144
+ f"Waiting for enumeration task poll loop to finish ({int(elapsed_time)}/{self.timeout} seconds)"
145
+ )
146
+
147
+ try:
148
+ # Wait for the task to complete or for 10 seconds, whichever comes first
149
+ await asyncio.wait_for(asyncio.shield(self.poll_task), timeout=10)
150
+ except asyncio.TimeoutError:
151
+ # This just means our 10-second check has elapsed, not that the task failed
152
+ pass
153
+
154
+ # Cancel the poll_task if it's still running
155
+ if not self.poll_task.done():
156
+ self.poll_task.cancel()
157
+ try:
158
+ await self.poll_task
159
+ except asyncio.CancelledError:
160
+ pass
bbot/modules/telerik.py CHANGED
@@ -173,8 +173,8 @@ class telerik(BaseModule):
173
173
  webresource = "Telerik.Web.UI.WebResource.axd?type=rau"
174
174
  result, _ = await self.test_detector(event.data, webresource)
175
175
  if result:
176
- if "RadAsyncUpload handler is registered succesfully" in result.text:
177
- self.debug(f"Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)")
176
+ if "RadAsyncUpload handler is registered successfully" in result.text:
177
+ self.debug("Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)")
178
178
 
179
179
  probe_data = {
180
180
  "rauPostData": (
@@ -216,7 +216,7 @@ class telerik(BaseModule):
216
216
  event,
217
217
  context=f"{{module}} scanned {event.data} and identified {{event.type}}: Telerik RAU AXD Handler",
218
218
  )
219
- if self.config.get("exploit_RAU_crypto") == True:
219
+ if self.config.get("exploit_RAU_crypto") is True:
220
220
  hostname = urlparse(event.data).netloc
221
221
  if hostname not in self.RAUConfirmed:
222
222
  self.RAUConfirmed.append(hostname)
@@ -270,7 +270,7 @@ class telerik(BaseModule):
270
270
  else:
271
271
  if "Cannot deserialize dialog parameters" in response.text:
272
272
  self.debug(f"Detected Telerik UI instance ({dh})")
273
- description = f"Telerik DialogHandler detected"
273
+ description = "Telerik DialogHandler detected"
274
274
  await self.emit_event(
275
275
  {"host": str(event.host), "url": f"{event.data}{dh}", "description": description},
276
276
  "FINDING",
@@ -289,8 +289,8 @@ class telerik(BaseModule):
289
289
  self.debug(validate_result)
290
290
  validate_status_code = getattr(validate_result, "status_code", 0)
291
291
  if validate_status_code not in (0, 500):
292
- self.debug(f"Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)")
293
- description = f"Telerik SpellCheckHandler detected"
292
+ self.debug("Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)")
293
+ description = "Telerik SpellCheckHandler detected"
294
294
  await self.emit_event(
295
295
  {
296
296
  "host": str(event.host),
@@ -334,7 +334,7 @@ class telerik(BaseModule):
334
334
  },
335
335
  "FINDING",
336
336
  event,
337
- context=f"{{module}} searched HTTP_RESPONSE and identified {{event.type}}: Telerik ChartImage AXD Handler",
337
+ context="{module} searched HTTP_RESPONSE and identified {event.type}: Telerik ChartImage AXD Handler",
338
338
  )
339
339
  elif '"_serializedConfiguration":"' in resp_body:
340
340
  await self.emit_event(
@@ -345,7 +345,7 @@ class telerik(BaseModule):
345
345
  },
346
346
  "FINDING",
347
347
  event,
348
- context=f"{{module}} searched HTTP_RESPONSE and identified {{event.type}}: Telerik AsyncUpload",
348
+ context="{module} searched HTTP_RESPONSE and identified {event.type}: Telerik AsyncUpload",
349
349
  )
350
350
 
351
351
  # Check for RAD Controls in URL
@@ -159,7 +159,7 @@ class bucket_template(BaseModule):
159
159
  valid = self.cloud_helper.is_valid_bucket_name(bucket_name)
160
160
  if valid and not self.helpers.is_ip(bucket_name):
161
161
  bucket_hash = hash(bucket_name)
162
- if not bucket_hash in self.buckets_tried:
162
+ if bucket_hash not in self.buckets_tried:
163
163
  self.buckets_tried.add(bucket_hash)
164
164
  return True
165
165
  return False
@@ -1,3 +1,5 @@
1
+ import traceback
2
+
1
3
  from bbot.modules.base import BaseModule
2
4
 
3
5
 
@@ -9,30 +11,36 @@ class github(BaseModule):
9
11
 
10
12
  _qsize = 1
11
13
  base_url = "https://api.github.com"
14
+ ping_url = f"{base_url}/zen"
15
+
16
+ def prepare_api_request(self, url, kwargs):
17
+ kwargs["headers"]["Authorization"] = f"token {self.api_key}"
18
+ return url, kwargs
12
19
 
13
20
  async def setup(self):
14
21
  await super().setup()
15
- self.api_key = None
16
22
  self.headers = {}
17
- for module_name in ("github", "github_codesearch", "github_org", "git_clone"):
18
- module_config = self.scan.config.get("modules", {}).get(module_name, {})
23
+ api_keys = set()
24
+ modules_config = self.scan.config.get("modules", {})
25
+ git_modules = [m for m in modules_config if str(m).startswith("git")]
26
+ for module_name in git_modules:
27
+ module_config = modules_config.get(module_name, {})
19
28
  api_key = module_config.get("api_key", "")
20
- if api_key:
21
- self.api_key = api_key
22
- self.headers = {"Authorization": f"token {self.api_key}"}
23
- break
24
- if not self.api_key:
29
+ if isinstance(api_key, str):
30
+ api_key = [api_key]
31
+ for key in api_key:
32
+ key = key.strip()
33
+ if key:
34
+ api_keys.add(key)
35
+ if not api_keys:
25
36
  if self.auth_required:
26
37
  return None, "No API key set"
38
+ self.api_key = api_keys
27
39
  try:
28
40
  await self.ping()
29
- self.hugesuccess(f"API is ready")
41
+ self.hugesuccess("API is ready")
30
42
  return True
31
43
  except Exception as e:
44
+ self.trace(traceback.format_exc())
32
45
  return None, f"Error with API ({str(e).strip()})"
33
46
  return True
34
-
35
- async def ping(self):
36
- url = f"{self.base_url}/zen"
37
- response = await self.helpers.request(url, headers=self.headers)
38
- assert getattr(response, "status_code", 0) == 200, response.text
@@ -0,0 +1,21 @@
1
+ from bbot.modules.base import BaseModule
2
+
3
+
4
+ class postman(BaseModule):
5
+ """
6
+ A template module for use of the GitHub API
7
+ Inherited by several other github modules.
8
+ """
9
+
10
+ base_url = "https://www.postman.com/_api"
11
+ api_url = "https://api.getpostman.com"
12
+ html_url = "https://www.postman.com"
13
+ ping_url = f"{api_url}/me"
14
+
15
+ headers = {
16
+ "Content-Type": "application/json",
17
+ "X-App-Version": "10.18.8-230926-0808",
18
+ "X-Entity-Team-Id": "0",
19
+ "Origin": "https://www.postman.com",
20
+ "Referer": "https://www.postman.com/search?q=&scope=public&type=all",
21
+ }