bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of bbot might be problematic. Click here for more details.

Files changed (278) hide show
  1. bbot/__init__.py +1 -1
  2. bbot/cli.py +3 -7
  3. bbot/core/config/files.py +0 -1
  4. bbot/core/config/logger.py +34 -4
  5. bbot/core/core.py +21 -4
  6. bbot/core/engine.py +9 -8
  7. bbot/core/event/base.py +131 -52
  8. bbot/core/helpers/bloom.py +10 -3
  9. bbot/core/helpers/command.py +8 -7
  10. bbot/core/helpers/depsinstaller/installer.py +31 -13
  11. bbot/core/helpers/diff.py +10 -10
  12. bbot/core/helpers/dns/brute.py +7 -4
  13. bbot/core/helpers/dns/dns.py +1 -2
  14. bbot/core/helpers/dns/engine.py +4 -6
  15. bbot/core/helpers/dns/helpers.py +2 -2
  16. bbot/core/helpers/dns/mock.py +0 -1
  17. bbot/core/helpers/files.py +1 -1
  18. bbot/core/helpers/helper.py +7 -4
  19. bbot/core/helpers/interactsh.py +3 -3
  20. bbot/core/helpers/libmagic.py +65 -0
  21. bbot/core/helpers/misc.py +65 -22
  22. bbot/core/helpers/names_generator.py +17 -3
  23. bbot/core/helpers/process.py +0 -20
  24. bbot/core/helpers/regex.py +1 -1
  25. bbot/core/helpers/regexes.py +12 -6
  26. bbot/core/helpers/validators.py +1 -2
  27. bbot/core/helpers/web/client.py +1 -1
  28. bbot/core/helpers/web/engine.py +1 -2
  29. bbot/core/helpers/web/web.py +4 -114
  30. bbot/core/helpers/wordcloud.py +5 -5
  31. bbot/core/modules.py +36 -27
  32. bbot/core/multiprocess.py +58 -0
  33. bbot/core/shared_deps.py +46 -3
  34. bbot/db/sql/models.py +147 -0
  35. bbot/defaults.yml +12 -10
  36. bbot/modules/anubisdb.py +2 -2
  37. bbot/modules/apkpure.py +63 -0
  38. bbot/modules/azure_tenant.py +2 -2
  39. bbot/modules/baddns.py +35 -19
  40. bbot/modules/baddns_direct.py +92 -0
  41. bbot/modules/baddns_zone.py +3 -8
  42. bbot/modules/badsecrets.py +4 -3
  43. bbot/modules/base.py +195 -51
  44. bbot/modules/bevigil.py +7 -7
  45. bbot/modules/binaryedge.py +7 -4
  46. bbot/modules/bufferoverrun.py +47 -0
  47. bbot/modules/builtwith.py +6 -10
  48. bbot/modules/bypass403.py +5 -5
  49. bbot/modules/c99.py +10 -7
  50. bbot/modules/censys.py +9 -13
  51. bbot/modules/certspotter.py +5 -3
  52. bbot/modules/chaos.py +9 -7
  53. bbot/modules/code_repository.py +1 -0
  54. bbot/modules/columbus.py +3 -3
  55. bbot/modules/crt.py +5 -3
  56. bbot/modules/deadly/dastardly.py +1 -1
  57. bbot/modules/deadly/ffuf.py +9 -9
  58. bbot/modules/deadly/nuclei.py +3 -3
  59. bbot/modules/deadly/vhost.py +4 -3
  60. bbot/modules/dehashed.py +1 -1
  61. bbot/modules/digitorus.py +1 -1
  62. bbot/modules/dnsbimi.py +145 -0
  63. bbot/modules/dnscaa.py +3 -3
  64. bbot/modules/dnsdumpster.py +4 -4
  65. bbot/modules/dnstlsrpt.py +144 -0
  66. bbot/modules/docker_pull.py +7 -5
  67. bbot/modules/dockerhub.py +2 -2
  68. bbot/modules/dotnetnuke.py +20 -21
  69. bbot/modules/emailformat.py +1 -1
  70. bbot/modules/extractous.py +122 -0
  71. bbot/modules/filedownload.py +9 -7
  72. bbot/modules/fullhunt.py +7 -4
  73. bbot/modules/generic_ssrf.py +5 -5
  74. bbot/modules/github_codesearch.py +3 -2
  75. bbot/modules/github_org.py +4 -4
  76. bbot/modules/github_workflows.py +4 -4
  77. bbot/modules/gitlab.py +2 -5
  78. bbot/modules/google_playstore.py +93 -0
  79. bbot/modules/gowitness.py +48 -50
  80. bbot/modules/hackertarget.py +5 -3
  81. bbot/modules/host_header.py +5 -5
  82. bbot/modules/httpx.py +1 -4
  83. bbot/modules/hunterio.py +3 -9
  84. bbot/modules/iis_shortnames.py +19 -30
  85. bbot/modules/internal/cloudcheck.py +29 -12
  86. bbot/modules/internal/dnsresolve.py +22 -22
  87. bbot/modules/internal/excavate.py +97 -59
  88. bbot/modules/internal/speculate.py +41 -32
  89. bbot/modules/internetdb.py +4 -2
  90. bbot/modules/ip2location.py +3 -5
  91. bbot/modules/ipneighbor.py +1 -1
  92. bbot/modules/ipstack.py +3 -8
  93. bbot/modules/jadx.py +87 -0
  94. bbot/modules/leakix.py +11 -10
  95. bbot/modules/myssl.py +2 -2
  96. bbot/modules/newsletters.py +2 -2
  97. bbot/modules/otx.py +5 -3
  98. bbot/modules/output/asset_inventory.py +7 -7
  99. bbot/modules/output/base.py +1 -1
  100. bbot/modules/output/csv.py +1 -1
  101. bbot/modules/output/http.py +20 -14
  102. bbot/modules/output/mysql.py +51 -0
  103. bbot/modules/output/neo4j.py +7 -2
  104. bbot/modules/output/postgres.py +49 -0
  105. bbot/modules/output/slack.py +0 -1
  106. bbot/modules/output/sqlite.py +29 -0
  107. bbot/modules/output/stdout.py +2 -2
  108. bbot/modules/output/teams.py +107 -6
  109. bbot/modules/paramminer_headers.py +8 -11
  110. bbot/modules/passivetotal.py +13 -13
  111. bbot/modules/portscan.py +32 -6
  112. bbot/modules/postman.py +50 -126
  113. bbot/modules/postman_download.py +220 -0
  114. bbot/modules/rapiddns.py +3 -8
  115. bbot/modules/report/asn.py +18 -11
  116. bbot/modules/robots.py +3 -3
  117. bbot/modules/securitytrails.py +7 -10
  118. bbot/modules/securitytxt.py +1 -1
  119. bbot/modules/shodan_dns.py +7 -9
  120. bbot/modules/sitedossier.py +1 -1
  121. bbot/modules/skymem.py +2 -2
  122. bbot/modules/social.py +2 -1
  123. bbot/modules/subdomaincenter.py +1 -1
  124. bbot/modules/subdomainradar.py +160 -0
  125. bbot/modules/telerik.py +8 -8
  126. bbot/modules/templates/bucket.py +1 -1
  127. bbot/modules/templates/github.py +22 -14
  128. bbot/modules/templates/postman.py +21 -0
  129. bbot/modules/templates/shodan.py +14 -13
  130. bbot/modules/templates/sql.py +95 -0
  131. bbot/modules/templates/subdomain_enum.py +51 -16
  132. bbot/modules/templates/webhook.py +2 -4
  133. bbot/modules/trickest.py +8 -37
  134. bbot/modules/trufflehog.py +10 -12
  135. bbot/modules/url_manipulation.py +3 -3
  136. bbot/modules/urlscan.py +1 -1
  137. bbot/modules/viewdns.py +1 -1
  138. bbot/modules/virustotal.py +8 -30
  139. bbot/modules/wafw00f.py +1 -1
  140. bbot/modules/wayback.py +1 -1
  141. bbot/modules/wpscan.py +17 -11
  142. bbot/modules/zoomeye.py +11 -6
  143. bbot/presets/baddns-thorough.yml +12 -0
  144. bbot/presets/fast.yml +16 -0
  145. bbot/presets/kitchen-sink.yml +1 -2
  146. bbot/presets/spider.yml +4 -0
  147. bbot/presets/subdomain-enum.yml +7 -7
  148. bbot/presets/web/dotnet-audit.yml +0 -1
  149. bbot/scanner/manager.py +5 -16
  150. bbot/scanner/preset/args.py +46 -26
  151. bbot/scanner/preset/environ.py +7 -2
  152. bbot/scanner/preset/path.py +7 -4
  153. bbot/scanner/preset/preset.py +36 -23
  154. bbot/scanner/scanner.py +172 -62
  155. bbot/scanner/target.py +236 -434
  156. bbot/scripts/docs.py +1 -1
  157. bbot/test/bbot_fixtures.py +13 -3
  158. bbot/test/conftest.py +132 -100
  159. bbot/test/fastapi_test.py +17 -0
  160. bbot/test/owasp_mastg.apk +0 -0
  161. bbot/test/run_tests.sh +4 -4
  162. bbot/test/test.conf +2 -0
  163. bbot/test/test_step_1/test__module__tests.py +0 -1
  164. bbot/test/test_step_1/test_bbot_fastapi.py +79 -0
  165. bbot/test/test_step_1/test_bloom_filter.py +2 -1
  166. bbot/test/test_step_1/test_cli.py +138 -64
  167. bbot/test/test_step_1/test_dns.py +61 -27
  168. bbot/test/test_step_1/test_engine.py +17 -19
  169. bbot/test/test_step_1/test_events.py +183 -30
  170. bbot/test/test_step_1/test_helpers.py +64 -29
  171. bbot/test/test_step_1/test_manager_deduplication.py +1 -1
  172. bbot/test/test_step_1/test_manager_scope_accuracy.py +333 -330
  173. bbot/test/test_step_1/test_modules_basic.py +68 -70
  174. bbot/test/test_step_1/test_presets.py +183 -100
  175. bbot/test/test_step_1/test_python_api.py +7 -2
  176. bbot/test/test_step_1/test_regexes.py +35 -5
  177. bbot/test/test_step_1/test_scan.py +39 -5
  178. bbot/test/test_step_1/test_scope.py +4 -3
  179. bbot/test/test_step_1/test_target.py +242 -145
  180. bbot/test/test_step_1/test_web.py +14 -10
  181. bbot/test/test_step_2/module_tests/base.py +15 -7
  182. bbot/test/test_step_2/module_tests/test_module_anubisdb.py +1 -1
  183. bbot/test/test_step_2/module_tests/test_module_apkpure.py +71 -0
  184. bbot/test/test_step_2/module_tests/test_module_asset_inventory.py +0 -1
  185. bbot/test/test_step_2/module_tests/test_module_azure_realm.py +1 -1
  186. bbot/test/test_step_2/module_tests/test_module_baddns.py +6 -6
  187. bbot/test/test_step_2/module_tests/test_module_baddns_direct.py +62 -0
  188. bbot/test/test_step_2/module_tests/test_module_bevigil.py +29 -2
  189. bbot/test/test_step_2/module_tests/test_module_binaryedge.py +4 -2
  190. bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py +2 -2
  191. bbot/test/test_step_2/module_tests/test_module_bucket_azure.py +1 -1
  192. bbot/test/test_step_2/module_tests/test_module_bufferoverrun.py +35 -0
  193. bbot/test/test_step_2/module_tests/test_module_builtwith.py +2 -2
  194. bbot/test/test_step_2/module_tests/test_module_bypass403.py +1 -1
  195. bbot/test/test_step_2/module_tests/test_module_c99.py +126 -0
  196. bbot/test/test_step_2/module_tests/test_module_censys.py +4 -1
  197. bbot/test/test_step_2/module_tests/test_module_cloudcheck.py +4 -0
  198. bbot/test/test_step_2/module_tests/test_module_code_repository.py +11 -1
  199. bbot/test/test_step_2/module_tests/test_module_columbus.py +1 -1
  200. bbot/test/test_step_2/module_tests/test_module_credshed.py +3 -3
  201. bbot/test/test_step_2/module_tests/test_module_dastardly.py +2 -1
  202. bbot/test/test_step_2/module_tests/test_module_dehashed.py +2 -2
  203. bbot/test/test_step_2/module_tests/test_module_digitorus.py +1 -1
  204. bbot/test/test_step_2/module_tests/test_module_discord.py +1 -1
  205. bbot/test/test_step_2/module_tests/test_module_dnsbimi.py +103 -0
  206. bbot/test/test_step_2/module_tests/test_module_dnsbrute.py +9 -10
  207. bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py +1 -2
  208. bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +1 -2
  209. bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py +4 -4
  210. bbot/test/test_step_2/module_tests/test_module_dnstlsrpt.py +64 -0
  211. bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +0 -8
  212. bbot/test/test_step_2/module_tests/test_module_excavate.py +28 -48
  213. bbot/test/test_step_2/module_tests/test_module_extractous.py +54 -0
  214. bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py +1 -1
  215. bbot/test/test_step_2/module_tests/test_module_filedownload.py +14 -14
  216. bbot/test/test_step_2/module_tests/test_module_git_clone.py +2 -2
  217. bbot/test/test_step_2/module_tests/test_module_github_org.py +19 -8
  218. bbot/test/test_step_2/module_tests/test_module_github_workflows.py +1 -1
  219. bbot/test/test_step_2/module_tests/test_module_gitlab.py +9 -4
  220. bbot/test/test_step_2/module_tests/test_module_google_playstore.py +83 -0
  221. bbot/test/test_step_2/module_tests/test_module_gowitness.py +4 -6
  222. bbot/test/test_step_2/module_tests/test_module_host_header.py +1 -1
  223. bbot/test/test_step_2/module_tests/test_module_http.py +4 -4
  224. bbot/test/test_step_2/module_tests/test_module_httpx.py +10 -8
  225. bbot/test/test_step_2/module_tests/test_module_hunterio.py +68 -4
  226. bbot/test/test_step_2/module_tests/test_module_jadx.py +55 -0
  227. bbot/test/test_step_2/module_tests/test_module_json.py +22 -9
  228. bbot/test/test_step_2/module_tests/test_module_leakix.py +7 -3
  229. bbot/test/test_step_2/module_tests/test_module_mysql.py +76 -0
  230. bbot/test/test_step_2/module_tests/test_module_myssl.py +1 -1
  231. bbot/test/test_step_2/module_tests/test_module_neo4j.py +1 -1
  232. bbot/test/test_step_2/module_tests/test_module_newsletters.py +16 -16
  233. bbot/test/test_step_2/module_tests/test_module_ntlm.py +8 -7
  234. bbot/test/test_step_2/module_tests/test_module_oauth.py +1 -1
  235. bbot/test/test_step_2/module_tests/test_module_otx.py +1 -1
  236. bbot/test/test_step_2/module_tests/test_module_paramminer_cookies.py +1 -2
  237. bbot/test/test_step_2/module_tests/test_module_paramminer_getparams.py +0 -6
  238. bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py +2 -9
  239. bbot/test/test_step_2/module_tests/test_module_passivetotal.py +3 -1
  240. bbot/test/test_step_2/module_tests/test_module_pgp.py +2 -2
  241. bbot/test/test_step_2/module_tests/test_module_portscan.py +9 -8
  242. bbot/test/test_step_2/module_tests/test_module_postgres.py +74 -0
  243. bbot/test/test_step_2/module_tests/test_module_postman.py +84 -253
  244. bbot/test/test_step_2/module_tests/test_module_postman_download.py +439 -0
  245. bbot/test/test_step_2/module_tests/test_module_rapiddns.py +93 -1
  246. bbot/test/test_step_2/module_tests/test_module_shodan_dns.py +20 -1
  247. bbot/test/test_step_2/module_tests/test_module_sitedossier.py +2 -2
  248. bbot/test/test_step_2/module_tests/test_module_smuggler.py +14 -14
  249. bbot/test/test_step_2/module_tests/test_module_social.py +11 -1
  250. bbot/test/test_step_2/module_tests/test_module_speculate.py +4 -8
  251. bbot/test/test_step_2/module_tests/test_module_splunk.py +4 -4
  252. bbot/test/test_step_2/module_tests/test_module_sqlite.py +18 -0
  253. bbot/test/test_step_2/module_tests/test_module_sslcert.py +1 -1
  254. bbot/test/test_step_2/module_tests/test_module_stdout.py +5 -3
  255. bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py +1 -1
  256. bbot/test/test_step_2/module_tests/test_module_subdomainradar.py +208 -0
  257. bbot/test/test_step_2/module_tests/test_module_subdomains.py +1 -1
  258. bbot/test/test_step_2/module_tests/test_module_teams.py +8 -6
  259. bbot/test/test_step_2/module_tests/test_module_telerik.py +1 -1
  260. bbot/test/test_step_2/module_tests/test_module_trufflehog.py +317 -14
  261. bbot/test/test_step_2/module_tests/test_module_viewdns.py +1 -1
  262. bbot/test/test_step_2/module_tests/test_module_wayback.py +1 -1
  263. bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py +2 -2
  264. bbot/wordlists/devops_mutations.txt +1 -1
  265. bbot/wordlists/ffuf_shortname_candidates.txt +1 -1
  266. bbot/wordlists/nameservers.txt +1 -1
  267. bbot/wordlists/paramminer_headers.txt +1 -1
  268. bbot/wordlists/paramminer_parameters.txt +1 -1
  269. bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt +1 -1
  270. bbot/wordlists/valid_url_schemes.txt +1 -1
  271. {bbot-2.0.1.4720rc0.dist-info → bbot-2.3.0.5401rc0.dist-info}/METADATA +48 -18
  272. bbot-2.3.0.5401rc0.dist-info/RECORD +421 -0
  273. {bbot-2.0.1.4720rc0.dist-info → bbot-2.3.0.5401rc0.dist-info}/WHEEL +1 -1
  274. bbot/modules/unstructured.py +0 -163
  275. bbot/test/test_step_2/module_tests/test_module_unstructured.py +0 -102
  276. bbot-2.0.1.4720rc0.dist-info/RECORD +0 -387
  277. {bbot-2.0.1.4720rc0.dist-info → bbot-2.3.0.5401rc0.dist-info}/LICENSE +0 -0
  278. {bbot-2.0.1.4720rc0.dist-info → bbot-2.3.0.5401rc0.dist-info}/entry_points.txt +0 -0
@@ -89,7 +89,6 @@ class TestParamminer_Getparams_boring_off(TestParamminer_Getparams):
89
89
  module_test.set_expect_requests(respond_args=respond_args)
90
90
 
91
91
  def check(self, module_test, events):
92
-
93
92
  emitted_boring_parameter = False
94
93
  for e in events:
95
94
  if e.type == "WEB_PARAMETER":
@@ -106,7 +105,6 @@ class TestParamminer_Getparams_boring_on(TestParamminer_Getparams_boring_off):
106
105
  }
107
106
 
108
107
  def check(self, module_test, events):
109
-
110
108
  emitted_boring_parameter = False
111
109
 
112
110
  for e in events:
@@ -160,15 +158,12 @@ class TestParamminer_Getparams_finish(Paramminer_Headers):
160
158
  module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
161
159
 
162
160
  def check(self, module_test, events):
163
-
164
161
  excavate_extracted_web_parameter = False
165
162
  found_hidden_getparam_recycled = False
166
163
  emitted_excavate_paramminer_duplicate = False
167
164
 
168
165
  for e in events:
169
-
170
166
  if e.type == "WEB_PARAMETER":
171
-
172
167
  if (
173
168
  "http://127.0.0.1:8888/test2.php" in e.data["url"]
174
169
  and "HTTP Extracted Parameter [abcd1234] (HTML Tags Submodule)" in e.data["description"]
@@ -213,7 +208,6 @@ class TestParamminer_Getparams_xmlspeculative(Paramminer_Headers):
213
208
  """
214
209
 
215
210
  async def setup_after_prep(self, module_test):
216
-
217
211
  module_test.scan.modules["paramminer_getparams"].rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA"
218
212
  module_test.monkeypatch.setattr(
219
213
  helper.HttpCompare, "gen_cache_buster", lambda *args, **kwargs: {"AAAAAA": "1"}
@@ -31,7 +31,7 @@ class Paramminer_Headers(ModuleTestBase):
31
31
  module_test.monkeypatch.setattr(
32
32
  helper.HttpCompare, "gen_cache_buster", lambda *args, **kwargs: {"AAAAAA": "1"}
33
33
  )
34
- expect_args = dict(headers={"tracestate": "AAAAAAAAAAAAAA"})
34
+ expect_args = {"headers": {"tracestate": "AAAAAAAAAAAAAA"}}
35
35
  respond_args = {"response_data": self.headers_body_match}
36
36
  module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
37
37
 
@@ -39,7 +39,6 @@ class Paramminer_Headers(ModuleTestBase):
39
39
  module_test.set_expect_requests(respond_args=respond_args)
40
40
 
41
41
  def check(self, module_test, events):
42
-
43
42
  found_reflected_header = False
44
43
  false_positive_match = False
45
44
 
@@ -60,7 +59,6 @@ class TestParamminer_Headers(Paramminer_Headers):
60
59
 
61
60
 
62
61
  class TestParamminer_Headers_noreflection(Paramminer_Headers):
63
-
64
62
  found_nonreflected_header = False
65
63
 
66
64
  headers_body_match = """
@@ -82,7 +80,6 @@ class TestParamminer_Headers_noreflection(Paramminer_Headers):
82
80
 
83
81
 
84
82
  class TestParamminer_Headers_extract(Paramminer_Headers):
85
-
86
83
  modules_overrides = ["httpx", "paramminer_headers", "excavate"]
87
84
  config_overrides = {
88
85
  "modules": {
@@ -115,7 +112,7 @@ class TestParamminer_Headers_extract(Paramminer_Headers):
115
112
  module_test.monkeypatch.setattr(
116
113
  helper.HttpCompare, "gen_cache_buster", lambda *args, **kwargs: {"AAAAAA": "1"}
117
114
  )
118
- expect_args = dict(headers={"foo": "AAAAAAAAAAAAAA"})
115
+ expect_args = {"headers": {"foo": "AAAAAAAAAAAAAA"}}
119
116
  respond_args = {"response_data": self.headers_body_match}
120
117
  module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
121
118
 
@@ -123,7 +120,6 @@ class TestParamminer_Headers_extract(Paramminer_Headers):
123
120
  module_test.set_expect_requests(respond_args=respond_args)
124
121
 
125
122
  def check(self, module_test, events):
126
-
127
123
  excavate_extracted_web_parameter = False
128
124
  used_recycled_parameter = False
129
125
 
@@ -139,17 +135,14 @@ class TestParamminer_Headers_extract(Paramminer_Headers):
139
135
 
140
136
 
141
137
  class TestParamminer_Headers_extract_norecycle(TestParamminer_Headers_extract):
142
-
143
138
  modules_overrides = ["httpx", "excavate"]
144
139
  config_overrides = {}
145
140
 
146
141
  async def setup_after_prep(self, module_test):
147
-
148
142
  respond_args = {"response_data": self.headers_body}
149
143
  module_test.set_expect_requests(respond_args=respond_args)
150
144
 
151
145
  def check(self, module_test, events):
152
-
153
146
  excavate_extracted_web_parameter = False
154
147
 
155
148
  for e in events:
@@ -2,15 +2,17 @@ from .base import ModuleTestBase
2
2
 
3
3
 
4
4
  class TestPassiveTotal(ModuleTestBase):
5
- config_overrides = {"modules": {"passivetotal": {"username": "jon@bls.fakedomain", "api_key": "asdf"}}}
5
+ config_overrides = {"modules": {"passivetotal": {"api_key": "jon@bls.fakedomain:asdf"}}}
6
6
 
7
7
  async def setup_before_prep(self, module_test):
8
8
  module_test.httpx_mock.add_response(
9
9
  url="https://api.passivetotal.org/v2/account/quota",
10
+ match_headers={"Authorization": "Basic am9uQGJscy5mYWtlZG9tYWluOmFzZGY="},
10
11
  json={"user": {"counts": {"search_api": 10}, "limits": {"search_api": 20}}},
11
12
  )
12
13
  module_test.httpx_mock.add_response(
13
14
  url="https://api.passivetotal.org/v2/enrichment/subdomains?query=blacklanternsecurity.com",
15
+ match_headers={"Authorization": "Basic am9uQGJscy5mYWtlZG9tYWluOmFzZGY="},
14
16
  json={"subdomains": ["asdf"]},
15
17
  )
16
18
 
@@ -9,10 +9,10 @@ class TestPGP(ModuleTestBase):
9
9
  <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
10
10
  <link href='/assets/css/pks.min.css' rel='stylesheet' type='text/css'>
11
11
  <style type="text/css">
12
-
12
+
13
13
  .uid { color: green; text-decoration: underline; }
14
14
  .warn { color: red; font-weight: bold; }
15
-
15
+
16
16
  </style></head><body><h1>Search results for 'blacklanternsecurity.com'</h1><pre>Type bits/keyID cr. time exp time key expir
17
17
  </pre>
18
18
 
@@ -21,7 +21,6 @@ class TestPortscan(ModuleTestBase):
21
21
  masscan_output_ping = """{ "ip": "8.8.8.8", "timestamp": "1719862594", "ports": [ {"port": 0, "proto": "icmp", "status": "open", "reason": "none", "ttl": 54} ] }"""
22
22
 
23
23
  async def setup_after_prep(self, module_test):
24
-
25
24
  from bbot.modules.base import BaseModule
26
25
 
27
26
  class DummyModule(BaseModule):
@@ -109,10 +108,12 @@ class TestPortscan(ModuleTestBase):
109
108
  if e.type == "DNS_NAME" and e.data == "dummy.asdf.evilcorp.net" and str(e.module) == "dummy_module"
110
109
  ]
111
110
  )
112
- assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 3
113
- assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 3
114
- assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 3
115
- assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 3
111
+ # the reason these numbers aren't exactly predictable is because we can't predict which one arrives first
112
+ # to the portscan module. Sometimes, one that would normally be deduped is force-emitted because it led to a new open port.
113
+ assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 4
114
+ assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 4
115
+ assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 4
116
+ assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 4
116
117
  assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443"])
117
118
  assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.5:80"])
118
119
  assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.6:631"])
@@ -121,7 +122,7 @@ class TestPortscan(ModuleTestBase):
121
122
  assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "asdf.evilcorp.net:80"])
122
123
  assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "dummy.asdf.evilcorp.net:80"])
123
124
  assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "dummy.evilcorp.com:631"])
124
- assert not any([e for e in events if e.type == "OPEN_TCP_PORT" and e.host == "dummy.www.evilcorp.com"])
125
+ assert not any(e for e in events if e.type == "OPEN_TCP_PORT" and e.host == "dummy.www.evilcorp.com")
125
126
 
126
127
 
127
128
  class TestPortscanPingFirst(TestPortscan):
@@ -135,7 +136,7 @@ class TestPortscanPingFirst(TestPortscan):
135
136
  assert self.ping_runs == 1
136
137
  open_port_events = [e for e in events if e.type == "OPEN_TCP_PORT"]
137
138
  assert len(open_port_events) == 3
138
- assert set([e.data for e in open_port_events]) == {"8.8.8.8:443", "evilcorp.com:443", "www.evilcorp.com:443"}
139
+ assert {e.data for e in open_port_events} == {"8.8.8.8:443", "evilcorp.com:443", "www.evilcorp.com:443"}
139
140
 
140
141
 
141
142
  class TestPortscanPingOnly(TestPortscan):
@@ -153,4 +154,4 @@ class TestPortscanPingOnly(TestPortscan):
153
154
  assert len(open_port_events) == 0
154
155
  ip_events = [e for e in events if e.type == "IP_ADDRESS"]
155
156
  assert len(ip_events) == 1
156
- assert set([e.data for e in ip_events]) == {"8.8.8.8"}
157
+ assert {e.data for e in ip_events} == {"8.8.8.8"}
@@ -0,0 +1,74 @@
1
+ import time
2
+ import asyncio
3
+
4
+ from .base import ModuleTestBase
5
+
6
+
7
+ class TestPostgres(ModuleTestBase):
8
+ targets = ["evilcorp.com"]
9
+ skip_distro_tests = True
10
+
11
+ async def setup_before_prep(self, module_test):
12
+ process = await asyncio.create_subprocess_exec(
13
+ "docker",
14
+ "run",
15
+ "--name",
16
+ "bbot-test-postgres",
17
+ "--rm",
18
+ "-e",
19
+ "POSTGRES_PASSWORD=bbotislife",
20
+ "-e",
21
+ "POSTGRES_USER=postgres",
22
+ "-p",
23
+ "5432:5432",
24
+ "-d",
25
+ "postgres",
26
+ )
27
+
28
+ import asyncpg
29
+
30
+ # wait for the container to start
31
+ start_time = time.time()
32
+ while True:
33
+ try:
34
+ # Connect to the default 'postgres' database to create 'bbot'
35
+ conn = await asyncpg.connect(
36
+ user="postgres", password="bbotislife", database="postgres", host="127.0.0.1"
37
+ )
38
+ await conn.execute("CREATE DATABASE bbot")
39
+ await conn.close()
40
+ break
41
+ except asyncpg.exceptions.DuplicateDatabaseError:
42
+ # If the database already exists, break the loop
43
+ break
44
+ except Exception as e:
45
+ if time.time() - start_time > 60: # timeout after 60 seconds
46
+ self.log.error("PostgreSQL server did not start in time.")
47
+ raise e
48
+ await asyncio.sleep(1)
49
+
50
+ if process.returncode != 0:
51
+ self.log.error("Failed to start PostgreSQL server")
52
+
53
+ async def check(self, module_test, events):
54
+ import asyncpg
55
+
56
+ # Connect to the PostgreSQL database
57
+ conn = await asyncpg.connect(user="postgres", password="bbotislife", database="bbot", host="127.0.0.1")
58
+
59
+ try:
60
+ events = await conn.fetch("SELECT * FROM event")
61
+ assert len(events) == 3, "No events found in PostgreSQL database"
62
+ scans = await conn.fetch("SELECT * FROM scan")
63
+ assert len(scans) == 1, "No scans found in PostgreSQL database"
64
+ targets = await conn.fetch("SELECT * FROM target")
65
+ assert len(targets) == 1, "No targets found in PostgreSQL database"
66
+ finally:
67
+ await conn.close()
68
+ process = await asyncio.create_subprocess_exec(
69
+ "docker", "stop", "bbot-test-postgres", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
70
+ )
71
+ stdout, stderr = await process.communicate()
72
+
73
+ if process.returncode != 0:
74
+ raise Exception(f"Failed to stop PostgreSQL server: {stderr.decode()}")
@@ -2,281 +2,112 @@ from .base import ModuleTestBase
2
2
 
3
3
 
4
4
  class TestPostman(ModuleTestBase):
5
- config_overrides = {
6
- "omit_event_types": [],
7
- "scope": {"report_distance": 1},
8
- }
9
-
10
- modules_overrides = ["postman", "httpx", "excavate"]
5
+ modules_overrides = ["postman", "speculate"]
11
6
 
12
7
  async def setup_after_prep(self, module_test):
8
+ await module_test.mock_dns(
9
+ {"blacklanternsecurity.com": {"A": ["127.0.0.99"]}, "github.com": {"A": ["127.0.0.99"]}}
10
+ )
13
11
  module_test.httpx_mock.add_response(
14
12
  url="https://www.postman.com/_api/ws/proxy",
15
13
  json={
16
14
  "data": [
17
15
  {
18
- "score": 499.22498,
19
- "normalizedScore": 8.43312276976538,
16
+ "score": 611.41156,
17
+ "normalizedScore": 23,
20
18
  "document": {
21
- "isPublisherVerified": False,
22
- "publisherType": "user",
23
- "curatedInList": [],
24
- "publisherId": "28329861",
25
- "publisherHandle": "",
26
- "publisherLogo": "",
27
- "isPublic": True,
28
- "customHostName": "",
29
- "id": "28329861-28329861-f6ef-4f23-9f3a-8431f3567ac1",
30
- "workspaces": [
31
- {
32
- "visibilityStatus": "public",
33
- "name": "BlackLanternSecuritySpilledSecrets",
34
- "id": "afa061be-9cb0-4520-9d4d-fe63361daf0f",
35
- "slug": "blacklanternsecurityspilledsecrets",
36
- }
37
- ],
38
- "collectionForkLabel": "",
39
- "method": "POST",
40
- "entityType": "request",
41
- "url": "www.example.com/index",
42
- "isBlacklisted": False,
43
- "warehouse__updated_at_collection": "2023-12-11 02:00:00",
44
- "isPrivateNetworkEntity": False,
45
- "warehouse__updated_at_request": "2023-12-11 02:00:00",
46
- "publisherName": "NA",
47
- "name": "A test post request",
48
- "privateNetworkMeta": "",
19
+ "watcherCount": 6,
20
+ "apiCount": 0,
21
+ "forkCount": 0,
22
+ "isblacklisted": "false",
23
+ "createdAt": "2021-06-15T14:03:51",
24
+ "publishertype": "team",
25
+ "publisherHandle": "blacklanternsecurity",
26
+ "id": "11498add-357d-4bc5-a008-0a2d44fb8829",
27
+ "slug": "bbot-public",
28
+ "updatedAt": "2024-07-30T11:00:35",
29
+ "entityType": "workspace",
30
+ "visibilityStatus": "public",
31
+ "forkcount": "0",
32
+ "tags": [],
33
+ "createdat": "2021-06-15T14:03:51",
34
+ "forkLabel": "",
35
+ "publisherName": "blacklanternsecurity",
36
+ "name": "BlackLanternSecurity BBOT [Public]",
37
+ "dependencyCount": 7,
38
+ "collectionCount": 6,
39
+ "warehouse__updated_at": "2024-07-30 11:00:00",
49
40
  "privateNetworkFolders": [],
50
- "documentType": "request",
51
- "collection": {
52
- "id": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
53
- "name": "Secret Collection",
54
- "tags": [],
55
- "forkCount": 0,
56
- "watcherCount": 0,
57
- "views": 31,
58
- "apiId": "",
59
- "apiName": "",
60
- },
61
- },
62
- },
63
- {
64
- "score": 498.22398,
65
- "normalizedScore": 8.43312266976538,
66
- "document": {
67
41
  "isPublisherVerified": False,
68
- "publisherType": "user",
42
+ "publisherType": "team",
69
43
  "curatedInList": [],
70
- "publisherId": "28329861",
71
- "publisherHandle": "",
44
+ "creatorId": "6900157",
45
+ "description": "",
46
+ "forklabel": "",
47
+ "publisherId": "299401",
72
48
  "publisherLogo": "",
49
+ "popularity": 5,
73
50
  "isPublic": True,
74
- "customHostName": "",
75
- "id": "b7fa2137-b7fa2137-23bf-45d1-b176-35359af30ded",
76
- "workspaces": [
77
- {
78
- "visibilityStatus": "public",
79
- "name": "SpilledSecrets",
80
- "id": "92d0451b-119d-4ef0-b74c-22c400e5ce05",
81
- "slug": "spilledsecrets",
82
- }
83
- ],
84
- "collectionForkLabel": "",
85
- "method": "POST",
86
- "entityType": "request",
87
- "url": "www.example.com/index",
51
+ "categories": [],
52
+ "universaltags": "",
53
+ "views": 5788,
54
+ "summary": "BLS public workspaces.",
55
+ "memberCount": 2,
88
56
  "isBlacklisted": False,
89
- "warehouse__updated_at_collection": "2023-12-11 02:00:00",
57
+ "publisherid": "299401",
90
58
  "isPrivateNetworkEntity": False,
91
- "warehouse__updated_at_request": "2023-12-11 02:00:00",
92
- "publisherName": "NA",
93
- "name": "A test post request",
59
+ "isDomainNonTrivial": True,
94
60
  "privateNetworkMeta": "",
95
- "privateNetworkFolders": [],
96
- "documentType": "request",
97
- "collection": {
98
- "id": "007e8d67-007e8d67-932b-46ff-b95c-a2aa216edaf3",
99
- "name": "Secret Collection",
100
- "tags": [],
101
- "forkCount": 0,
102
- "watcherCount": 0,
103
- "views": 31,
104
- "apiId": "",
105
- "apiName": "",
106
- },
61
+ "updatedat": "2021-10-20T16:19:29",
62
+ "documentType": "workspace",
107
63
  },
108
- },
64
+ "highlight": {"summary": "<b>BLS</b> BBOT api test."},
65
+ }
109
66
  ],
110
- },
111
- )
112
- module_test.httpx_mock.add_response(
113
- url="https://www.postman.com/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f",
114
- json={
115
- "model_id": "afa061be-9cb0-4520-9d4d-fe63361daf0f",
116
- "meta": {"model": "workspace", "action": "find"},
117
- "data": {
118
- "id": "afa061be-9cb0-4520-9d4d-fe63361daf0f",
119
- "name": "SpilledSecrets",
120
- "description": "A Mock workspace environment filled with fake secrets to act as a testing ground for secret scanners",
121
- "summary": "A Public workspace with mock secrets",
122
- "createdBy": "28329861",
123
- "updatedBy": "28329861",
124
- "team": "28329861",
125
- "createdAt": "2023-12-13T19:12:21.000Z",
126
- "updatedAt": "2023-12-13T19:15:07.000Z",
127
- "visibilityStatus": "public",
128
- "profileInfo": {
129
- "slug": "spilledsecrets",
130
- "profileType": "team",
131
- "profileId": "28329861",
132
- "publicHandle": "https://www.postman.com/lunar-pizza-28329861",
133
- "publicImageURL": "https://res.cloudinary.com/postman/image/upload/t_team_logo/v1/team/default-L4",
134
- "publicName": "lunar-pizza-28329861",
135
- "isVerified": False,
67
+ "meta": {
68
+ "queryText": "blacklanternsecurity",
69
+ "total": {
70
+ "collection": 0,
71
+ "request": 0,
72
+ "workspace": 1,
73
+ "api": 0,
74
+ "team": 0,
75
+ "user": 0,
76
+ "flow": 0,
77
+ "apiDefinition": 0,
78
+ "privateNetworkFolder": 0,
136
79
  },
137
- "user": "28329861",
138
- "type": "team",
139
- "dependencies": {
140
- "collections": ["28129865-d9f8833b-3dd2-4b07-9634-1831206d5205"],
141
- "environments": ["28129865-fa7edca0-2df6-4187-9805-11845912f567"],
142
- "globals": ["28735eff-5ecd-43e6-8473-387f160f220f"],
80
+ "state": "AQ4",
81
+ "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None},
82
+ "featureFlags": {
83
+ "enabledPublicResultCuration": True,
84
+ "boostByPopularity": True,
85
+ "reRankPostNormalization": True,
86
+ "enableUrlBarHostNameSearch": True,
143
87
  },
144
- "members": {"users": {"28129865": {"id": "28129865"}}},
145
88
  },
146
89
  },
147
90
  )
148
- module_test.httpx_mock.add_response(
149
- url="https://www.postman.com/_api/list/collection?workspace=afa061be-9cb0-4520-9d4d-fe63361daf0f",
150
- json={
151
- "data": [
152
- {
153
- "id": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
154
- "name": "Secret Collection",
155
- "folders_order": ["d6dd1092-94d4-462c-be48-4e3cad3b8612"],
156
- "order": ["67c9db4c-d0ed-461c-86d2-9a8c5a5de896"],
157
- "attributes": {
158
- "permissions": {
159
- "userCanUpdate": False,
160
- "userCanDelete": False,
161
- "userCanShare": False,
162
- "userCanCreateMock": False,
163
- "userCanCreateMonitor": False,
164
- "anybodyCanView": True,
165
- "teamCanView": True,
166
- },
167
- "fork": None,
168
- "parent": {"type": "workspace", "id": "afa061be-9cb0-4520-9d4d-fe63361daf0f"},
169
- "flags": {"isArchived": False, "isFavorite": False},
170
- },
171
- "folders": [
172
- {
173
- "id": "28129865-d6dd1092-94d4-462c-be48-4e3cad3b8612",
174
- "name": "Nested folder",
175
- "folder": None,
176
- "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
177
- "folders_order": ["73a999c1-4246-4805-91bd-0232cc75958a"],
178
- "order": ["3aa78b71-2c4f-4299-94df-287ed1036409"],
179
- "folders": [
180
- {
181
- "id": "28129865-73a999c1-4246-4805-91bd-0232cc75958a",
182
- "name": "Another Nested Folder",
183
- "folder": "28129865-d6dd1092-94d4-462c-be48-4e3cad3b8612",
184
- "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
185
- "folders_order": [],
186
- "order": ["987c8ac8-bfa9-4bab-ade9-88ccf0597862"],
187
- "folders": [],
188
- "requests": [
189
- {
190
- "id": "28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862",
191
- "name": "Delete User",
192
- "method": "DELETE",
193
- "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
194
- "folder": "28129865-73a999c1-4246-4805-91bd-0232cc75958a",
195
- "responses_order": [],
196
- "responses": [],
197
- }
198
- ],
199
- }
200
- ],
201
- "requests": [
202
- {
203
- "id": "28129865-3aa78b71-2c4f-4299-94df-287ed1036409",
204
- "name": "Login",
205
- "method": "POST",
206
- "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
207
- "folder": "28129865-d6dd1092-94d4-462c-be48-4e3cad3b8612",
208
- "responses_order": [],
209
- "responses": [],
210
- }
211
- ],
212
- }
213
- ],
214
- "requests": [
215
- {
216
- "id": "28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896",
217
- "name": "Example Basic HTTP request",
218
- "method": "GET",
219
- "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
220
- "folder": None,
221
- "responses_order": [],
222
- "responses": [],
223
- }
224
- ],
225
- }
226
- ]
227
- },
228
- )
229
-
230
- old_emit_event = module_test.module.emit_event
231
-
232
- async def new_emit_event(event_data, event_type, **kwargs):
233
- if event_data.startswith("https://www.postman.com"):
234
- event_data = event_data.replace("https://www.postman.com", "http://127.0.0.1:8888")
235
- await old_emit_event(event_data, event_type, **kwargs)
236
-
237
- module_test.monkeypatch.setattr(module_test.module, "emit_event", new_emit_event)
238
- await module_test.mock_dns(
239
- {"blacklanternsecurity.com": {"A": ["127.0.0.1"]}, "asdf.blacklanternsecurity.com": {"A": ["127.0.0.1"]}}
240
- )
241
-
242
- request_args = dict(uri="/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862")
243
- respond_args = dict(response_data="https://asdf.blacklanternsecurity.com")
244
- module_test.set_expect_requests(request_args, respond_args)
245
91
 
246
92
  def check(self, module_test, events):
247
- assert any(
248
- e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f" for e in events
249
- ), "Failed to detect workspace"
250
- assert any(
251
- e.data != "http://127.0.0.1:8888/_api/workspace/92d0451b-119d-4ef0-b74c-22c400e5ce05" for e in events
252
- ), "Workspace should not be detected"
253
- assert any(
254
- e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f/globals"
255
- for e in events
256
- ), "Failed to detect workspace globals"
257
- assert any(
258
- e.data == "http://127.0.0.1:8888/_api/environment/28129865-fa7edca0-2df6-4187-9805-11845912f567"
259
- for e in events
260
- ), "Failed to detect workspace environment"
261
- assert any(
262
- e.data == "http://127.0.0.1:8888/_api/collection/28129865-d9f8833b-3dd2-4b07-9634-1831206d5205"
263
- for e in events
264
- ), "Failed to detect collection"
265
- assert any(
266
- e.data == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862"
267
- for e in events
268
- ), "Failed to detect collection request #1"
269
- assert any(
270
- e.data == "http://127.0.0.1:8888/_api/request/28129865-3aa78b71-2c4f-4299-94df-287ed1036409"
271
- for e in events
272
- ), "Failed to detect collection request #2"
273
- assert any(
274
- e.data == "http://127.0.0.1:8888/_api/request/28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896"
275
- for e in events
276
- ), "Failed to detect collection request #3"
277
- assert any(
278
- e.type == "HTTP_RESPONSE"
279
- and e.data["url"] == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862"
280
- for e in events
281
- ), "Failed to emit HTTP_RESPONSE"
282
- assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"
93
+ assert len(events) == 5
94
+ assert 1 == len(
95
+ [
96
+ e
97
+ for e in events
98
+ if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com" and e.scope_distance == 0
99
+ ]
100
+ ), "Failed to emit target DNS_NAME"
101
+ assert 1 == len(
102
+ [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 0]
103
+ ), "Failed to find ORG_STUB"
104
+ assert 1 == len(
105
+ [
106
+ e
107
+ for e in events
108
+ if e.type == "CODE_REPOSITORY"
109
+ and "postman" in e.tags
110
+ and e.data["url"] == "https://www.postman.com/blacklanternsecurity/bbot-public"
111
+ and e.scope_distance == 1
112
+ ]
113
+ ), "Failed to find blacklanternsecurity postman workspace"