bbot 2.4.2.6677rc0__py3-none-any.whl → 2.5.0.6715rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of bbot might be problematic. Click here for more details.

Files changed (42) hide show
  1. bbot/__init__.py +1 -1
  2. bbot/core/helpers/async_helpers.py +29 -8
  3. bbot/defaults.yml +6 -0
  4. bbot/modules/apkpure.py +5 -3
  5. bbot/modules/base.py +86 -32
  6. bbot/modules/docker_pull.py +3 -3
  7. bbot/modules/filedownload.py +8 -4
  8. bbot/modules/git_clone.py +5 -2
  9. bbot/modules/gitdumper.py +14 -5
  10. bbot/modules/github_workflows.py +12 -5
  11. bbot/modules/lightfuzz/lightfuzz.py +1 -1
  12. bbot/modules/lightfuzz/submodules/serial.py +11 -1
  13. bbot/modules/lightfuzz/submodules/sqli.py +1 -0
  14. bbot/modules/lightfuzz/submodules/xss.py +4 -4
  15. bbot/modules/nuclei.py +4 -2
  16. bbot/modules/portscan.py +2 -0
  17. bbot/modules/postman_download.py +6 -3
  18. bbot/modules/trufflehog.py +1 -2
  19. bbot/presets/web/lightfuzz-heavy.yml +1 -1
  20. bbot/presets/web/lightfuzz-medium.yml +1 -1
  21. bbot/presets/web/lightfuzz-superheavy.yml +1 -1
  22. bbot/scanner/preset/args.py +10 -1
  23. bbot/scanner/preset/preset.py +0 -2
  24. bbot/scanner/scanner.py +1 -4
  25. bbot/scripts/docs.py +8 -0
  26. bbot/test/test_step_1/test_scan.py +57 -0
  27. bbot/test/test_step_2/module_tests/test_module_apkpure.py +2 -0
  28. bbot/test/test_step_2/module_tests/test_module_bucket_file_enum.py +8 -3
  29. bbot/test/test_step_2/module_tests/test_module_docker_pull.py +2 -0
  30. bbot/test/test_step_2/module_tests/test_module_filedownload.py +5 -1
  31. bbot/test/test_step_2/module_tests/test_module_git_clone.py +4 -1
  32. bbot/test/test_step_2/module_tests/test_module_gitdumper.py +2 -0
  33. bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +2 -130
  34. bbot/test/test_step_2/module_tests/test_module_portscan.py +3 -3
  35. bbot/test/test_step_2/module_tests/test_module_postman_download.py +6 -1
  36. bbot/test/test_step_2/module_tests/test_module_trufflehog.py +38 -12
  37. {bbot-2.4.2.6677rc0.dist-info → bbot-2.5.0.6715rc0.dist-info}/METADATA +1 -1
  38. {bbot-2.4.2.6677rc0.dist-info → bbot-2.5.0.6715rc0.dist-info}/RECORD +41 -42
  39. bbot/modules/lightfuzz/submodules/nosqli.py +0 -183
  40. {bbot-2.4.2.6677rc0.dist-info → bbot-2.5.0.6715rc0.dist-info}/LICENSE +0 -0
  41. {bbot-2.4.2.6677rc0.dist-info → bbot-2.5.0.6715rc0.dist-info}/WHEEL +0 -0
  42. {bbot-2.4.2.6677rc0.dist-info → bbot-2.5.0.6715rc0.dist-info}/entry_points.txt +0 -0
@@ -12,5 +12,5 @@ modules:
12
12
  config:
13
13
  modules:
14
14
  lightfuzz:
15
- enabled_submodules: [cmdi,crypto,nosqli,path,serial,sqli,ssti,xss]
15
+ enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss]
16
16
  disable_post: False
@@ -11,4 +11,4 @@ modules:
11
11
  config:
12
12
  modules:
13
13
  lightfuzz:
14
- enabled_submodules: [cmdi,crypto,nosqli,path,serial,sqli,ssti,xss]
14
+ enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss]
@@ -8,6 +8,6 @@ config:
8
8
  modules:
9
9
  lightfuzz:
10
10
  force_common_headers: True # Fuzz common headers like X-Forwarded-For even if they're not observed on the target
11
- enabled_submodules: [cmdi,crypto,nosqli,path,serial,sqli,ssti,xss]
11
+ enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss]
12
12
  excavate:
13
13
  speculate_params: True # speculate potential parameters extracted from JSON/XML web responses
@@ -9,9 +9,18 @@ from bbot.core.helpers.misc import chain_lists, get_closest_match, get_keys_in_d
9
9
  log = logging.getLogger("bbot.presets.args")
10
10
 
11
11
 
12
+ universal_module_options = {
13
+ "batch_size": "The number of events to process in a single batch (only applies to batch modules)",
14
+ "module_threads": "How many event handlers to run in parallel",
15
+ "module_timeout": "Max time in seconds to spend handling each event or batch of events",
16
+ }
17
+
18
+
12
19
  class BBOTArgs:
13
20
  # module config options to exclude from validation
14
- exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|module_threads)$")
21
+ exclude_from_validation = re.compile(
22
+ r".*modules\.[a-z0-9_]+\.(?:" + "|".join(universal_module_options.keys()) + ")$"
23
+ )
15
24
 
16
25
  scan_examples = [
17
26
  (
@@ -276,8 +276,6 @@ class Preset(metaclass=BasePreset):
276
276
  self.exclude_flags.update(set(exclude_flags))
277
277
  self.require_flags.update(set(require_flags))
278
278
 
279
- # log.critical(f"{self.name}: verbose: {self.verbose}, debug: {self.debug}, silent: {self.silent}")
280
-
281
279
  @property
282
280
  def bbot_home(self):
283
281
  return Path(self.config.get("home", "~/.bbot")).expanduser().resolve()
bbot/scanner/scanner.py CHANGED
@@ -245,8 +245,6 @@ class Scanner:
245
245
  self._cleanedup = False
246
246
  self._omitted_event_types = None
247
247
 
248
- self.__loop = None
249
- self._manager_worker_loop_tasks = []
250
248
  self.init_events_task = None
251
249
  self.ticker_task = None
252
250
  self.dispatcher_tasks = []
@@ -726,6 +724,7 @@ class Scanner:
726
724
  scan_active_status.append(f" - {task}:")
727
725
  # scan_active_status.append(f" incoming_queue_size: {m.num_incoming_events}")
728
726
  # scan_active_status.append(f" outgoing_queue_size: {m.outgoing_event_queue.qsize()}")
727
+
729
728
  for line in scan_active_status:
730
729
  self.debug(line)
731
730
 
@@ -834,8 +833,6 @@ class Scanner:
834
833
  tasks.append(self.ticker_task)
835
834
  # dispatcher
836
835
  tasks += self.dispatcher_tasks
837
- # manager worker loops
838
- tasks += self._manager_worker_loop_tasks
839
836
  self.helpers.cancel_tasks_sync(tasks)
840
837
  # process pool
841
838
  self.helpers.process_pool.shutdown(cancel_futures=True)
bbot/scripts/docs.py CHANGED
@@ -181,6 +181,14 @@ def update_docs():
181
181
  assert len(bbot_output_module_table.splitlines()) > 10
182
182
  update_md_files("BBOT OUTPUT MODULES", bbot_output_module_table)
183
183
 
184
+ # BBOT universal module options
185
+ from bbot.scanner.preset.args import universal_module_options
186
+
187
+ universal_module_options_table = ""
188
+ for option, description in universal_module_options.items():
189
+ universal_module_options_table += f"**{option}**: {description}\n"
190
+ update_md_files("BBOT UNIVERSAL MODULE OPTIONS", universal_module_options_table)
191
+
184
192
  # BBOT module options
185
193
  bbot_module_options_table = DEFAULT_PRESET.module_loader.modules_options_table()
186
194
  assert len(bbot_module_options_table.splitlines()) > 100
@@ -88,6 +88,63 @@ async def test_scan(
88
88
  assert len(scan6.dns_strings) == 1
89
89
 
90
90
 
91
+ @pytest.mark.asyncio
92
+ async def test_task_scan_handle_event_timeout(bbot_scanner):
93
+ from bbot.modules.base import BaseModule
94
+
95
+ # make a module that takes a long time to handle an event
96
+ class LongModule(BaseModule):
97
+ watched_events = ["IP_ADDRESS"]
98
+ handled_event = False
99
+ cancelled = False
100
+ _name = "long"
101
+
102
+ async def handle_event(self, event):
103
+ self.handled_event = True
104
+ try:
105
+ await self.helpers.sleep(99999999)
106
+ except asyncio.CancelledError:
107
+ self.cancelled = True
108
+ raise
109
+
110
+ # same thing but handle_batch
111
+ class LongBatchModule(BaseModule):
112
+ watched_events = ["IP_ADDRESS"]
113
+ handled_event = False
114
+ canceled = False
115
+ _name = "long_batch"
116
+ _batch_size = 2
117
+
118
+ async def handle_batch(self, *events):
119
+ self.handled_event = True
120
+ try:
121
+ await self.helpers.sleep(99999999)
122
+ except asyncio.CancelledError:
123
+ self.cancelled = True
124
+ raise
125
+
126
+ # scan with both modules
127
+ scan = bbot_scanner(
128
+ "127.0.0.1",
129
+ config={
130
+ "module_handle_event_timeout": 5,
131
+ "module_handle_batch_timeout": 5,
132
+ },
133
+ )
134
+ await scan._prep()
135
+ scan.modules["long"] = LongModule(scan=scan)
136
+ scan.modules["long_batch"] = LongBatchModule(scan=scan)
137
+ events = [e async for e in scan.async_start()]
138
+ assert events
139
+ assert any(e.data == "127.0.0.1" for e in events)
140
+ # make sure both modules were called
141
+ assert scan.modules["long"].handled_event
142
+ assert scan.modules["long_batch"].handled_event
143
+ # they should also be cancelled
144
+ assert scan.modules["long"].cancelled
145
+ assert scan.modules["long_batch"].cancelled
146
+
147
+
91
148
  @pytest.mark.asyncio
92
149
  async def test_url_extension_handling(bbot_scanner):
93
150
  scan = bbot_scanner(config={"url_extension_blacklist": ["css"], "url_extension_httpx_only": ["js"]})
@@ -1,9 +1,11 @@
1
1
  from pathlib import Path
2
2
  from .base import ModuleTestBase, tempapkfile
3
+ from bbot.test.bbot_fixtures import bbot_test_dir
3
4
 
4
5
 
5
6
  class TestAPKPure(ModuleTestBase):
6
7
  modules_overrides = ["apkpure", "google_playstore", "speculate"]
8
+ config_overrides = {"modules": {"apkpure": {"output_folder": str(bbot_test_dir / "test_apkpure_files")}}}
7
9
  apk_file = tempapkfile()
8
10
 
9
11
  async def setup_after_prep(self, module_test):
@@ -1,10 +1,16 @@
1
1
  from .base import ModuleTestBase
2
+ from bbot.test.bbot_fixtures import bbot_test_dir
2
3
 
3
4
 
4
5
  class TestBucket_File_Enum(ModuleTestBase):
5
6
  targets = ["http://127.0.0.1:8888"]
6
7
  modules_overrides = ["bucket_file_enum", "filedownload", "httpx", "excavate", "cloudcheck"]
7
- config_overrides = {"scope": {"report_distance": 5}}
8
+
9
+ download_dir = bbot_test_dir / "test_bucket_file_enum"
10
+ config_overrides = {
11
+ "scope": {"report_distance": 5},
12
+ "modules": {"filedownload": {"output_folder": str(download_dir)}},
13
+ }
8
14
 
9
15
  open_bucket_url = "https://testbucket.s3.amazonaws.com/"
10
16
  open_bucket_body = """<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>testbucket</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>index.html</Key><LastModified>2023-05-22T23:04:38.000Z</LastModified><ETag>&quot;4a2d2d114f3abf90f8bd127c1f25095a&quot;</ETag><Size>5</Size><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>test.pdf</Key><LastModified>2022-04-30T21:13:40.000Z</LastModified><ETag>&quot;723b0018c2f5a7ef06a34f84f6fa97e4&quot;</ETag><Size>388901</Size><StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>"""
@@ -32,8 +38,7 @@ trailer <</Root 1 0 R>>"""
32
38
  )
33
39
 
34
40
  def check(self, module_test, events):
35
- download_dir = module_test.scan.home / "filedownload"
36
- files = list(download_dir.glob("*.pdf"))
41
+ files = list((self.download_dir / "filedownload").glob("*.pdf"))
37
42
  assert any(e.type == "URL_UNVERIFIED" and e.data.endswith("test.pdf") for e in events)
38
43
  assert not any(e.type == "URL_UNVERIFIED" and e.data.endswith("test.css") for e in events)
39
44
  assert any(f.name.endswith("test.pdf") for f in files), "Failed to download PDF file from open bucket"
@@ -3,10 +3,12 @@ import tarfile
3
3
  from pathlib import Path
4
4
 
5
5
  from .base import ModuleTestBase
6
+ from bbot.test.bbot_fixtures import bbot_test_dir
6
7
 
7
8
 
8
9
  class TestDockerPull(ModuleTestBase):
9
10
  modules_overrides = ["speculate", "dockerhub", "docker_pull"]
11
+ config_overrides = {"modules": {"docker_pull": {"output_folder": str(bbot_test_dir / "test_docker_files")}}}
10
12
 
11
13
  async def setup_before_prep(self, module_test):
12
14
  module_test.httpx_mock.add_response(
@@ -1,11 +1,15 @@
1
1
  from pathlib import Path
2
2
  from .base import ModuleTestBase
3
+ from bbot.test.bbot_fixtures import bbot_test_dir
3
4
 
4
5
 
5
6
  class TestFileDownload(ModuleTestBase):
6
7
  targets = ["http://127.0.0.1:8888"]
7
8
  modules_overrides = ["filedownload", "httpx", "excavate", "speculate"]
8
- config_overrides = {"web": {"spider_distance": 2, "spider_depth": 2}}
9
+ config_overrides = {
10
+ "web": {"spider_distance": 2, "spider_depth": 2},
11
+ "modules": {"filedownload": {"output_folder": str(bbot_test_dir / "test_filedownload_files")}},
12
+ }
9
13
 
10
14
  pdf_data = """%PDF-1.
11
15
  1 0 obj<</Pages 2 0 R>>endobj
@@ -6,10 +6,13 @@ import subprocess
6
6
  from pathlib import Path
7
7
 
8
8
  from .base import ModuleTestBase
9
+ from bbot.test.bbot_fixtures import bbot_test_dir
9
10
 
10
11
 
11
12
  class TestGit_Clone(ModuleTestBase):
12
- config_overrides = {"modules": {"git_clone": {"api_key": "asdf"}}}
13
+ config_overrides = {
14
+ "modules": {"git_clone": {"api_key": "asdf", "output_folder": str(bbot_test_dir / "test_git_files")}}
15
+ }
13
16
  modules_overrides = ["github_org", "speculate", "git_clone"]
14
17
 
15
18
  file_content = "https://admin:admin@the-internet.herokuapp.com/basic_auth"
@@ -1,5 +1,6 @@
1
1
  from pathlib import Path
2
2
  from .base import ModuleTestBase
3
+ from bbot.test.bbot_fixtures import bbot_test_dir
3
4
 
4
5
 
5
6
  class TestGitDumper_Dirlisting(ModuleTestBase):
@@ -8,6 +9,7 @@ class TestGitDumper_Dirlisting(ModuleTestBase):
8
9
  ]
9
10
 
10
11
  modules_overrides = ["git", "gitdumper", "httpx"]
12
+ config_overrides = {"modules": {"gitdumper": {"output_folder": str(bbot_test_dir / "test_output")}}}
11
13
 
12
14
  index_html = """<html>
13
15
  <head>
@@ -639,132 +639,6 @@ class Test_Lightfuzz_urlencoding(Test_Lightfuzz_xss_injs):
639
639
  assert xss_finding_emitted, "In Javascript XSS FINDING not emitted"
640
640
 
641
641
 
642
- class Test_Lightfuzz_nosqli_quoteescape(ModuleTestBase):
643
- targets = ["http://127.0.0.1:8888"]
644
- modules_overrides = ["httpx", "lightfuzz", "excavate"]
645
- config_overrides = {
646
- "interactsh_disable": True,
647
- "modules": {
648
- "lightfuzz": {
649
- "enabled_submodules": ["nosqli"],
650
- }
651
- },
652
- }
653
-
654
- def request_handler(self, request):
655
- normal_block = """
656
- <section class="search-filters">
657
- <label>Refine your search:</label>
658
- <a class="filter-category" href="/?category=Pets">Pets</a>
659
- </section>
660
- """
661
-
662
- qs = str(request.query_string.decode())
663
- if "category=" in qs:
664
- value = qs.split("=")[1]
665
- if "&" in value:
666
- value = value.split("&")[0]
667
- if value == "Pets%27":
668
- return Response("JSON ERROR!", status=500)
669
- elif value == "Pets%5C%27":
670
- return Response("No results", status=200)
671
- elif value == "Pets%27%20%26%26%200%20%26%26%20%27x":
672
- return Response("No results", status=200)
673
- elif value == "Pets%27%20%26%26%201%20%26%26%20%27x":
674
- return Response('{"category":"Pets","entries":["dog","cat","bird"]}', status=200)
675
- else:
676
- return Response("No results", status=200)
677
- return Response(normal_block, status=200)
678
-
679
- async def setup_after_prep(self, module_test):
680
- module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA"
681
- expect_args = re.compile("/")
682
- module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)
683
-
684
- def check(self, module_test, events):
685
- nosqli_finding_emitted = False
686
- finding_count = 0
687
- for e in events:
688
- if e.type == "FINDING":
689
- finding_count += 1
690
- if (
691
- "Possible NoSQL Injection. Parameter: [category] Parameter Type: [GETPARAM] Original Value: [Pets] Detection Method: [Quote/Escaped Quote + Conditional Affect]"
692
- in e.data["description"]
693
- ):
694
- nosqli_finding_emitted = True
695
- assert nosqli_finding_emitted, "NoSQLi FINDING not emitted"
696
- assert finding_count == 1, "Unexpected FINDING events reported"
697
-
698
-
699
- class Test_Lightfuzz_nosqli_negation(Test_Lightfuzz_nosqli_quoteescape):
700
- def request_handler(self, request):
701
- form_block = """
702
- <form method="POST" action="">
703
- <label for="username">Username:</label>
704
- <input type="text" id="username" name="username" required>
705
- <br>
706
- <label for="password">Password:</label>
707
- <input type="password" id="password" name="password" required>
708
- <br>
709
- <button type="submit">Login</button>
710
- </form>
711
- """
712
- if request.method == "GET":
713
- return Response(form_block, status=200)
714
-
715
- if "username[$ne]" in request.form.keys() and "password[$ne]" in request.form.keys():
716
- return Response("Welcome, testuser1!", status=200)
717
- if "username[$eq]" in request.form.keys() and "password[$eq]" in request.form.keys():
718
- return Response("Invalid Username or Password!", status=200)
719
- else:
720
- return Response("Invalid Username or Password!", status=200)
721
-
722
- def check(self, module_test, events):
723
- nosqli_finding_emitted = False
724
- finding_count = 0
725
- for e in events:
726
- if e.type == "FINDING":
727
- finding_count += 1
728
- if (
729
- "Possible NoSQL Injection. Parameter: [password] Parameter Type: [POSTPARAM] Detection Method: [Parameter Name Operator Injection - Negation ([$ne])] Differences: [body]"
730
- in e.data["description"]
731
- ):
732
- nosqli_finding_emitted = True
733
- assert nosqli_finding_emitted, "NoSQLi FINDING not emitted"
734
- assert finding_count == 2, "Unexpected FINDING events reported"
735
-
736
-
737
- class Test_Lightfuzz_nosqli_negation_falsepositive(Test_Lightfuzz_nosqli_quoteescape):
738
- def request_handler(self, request):
739
- form_block = """
740
- <form method="POST" action="">
741
- <label for="username">Username:</label>
742
- <input type="text" id="username" name="username" required>
743
- <br>
744
- <label for="password">Password:</label>
745
- <input type="password" id="password" name="password" required>
746
- <br>
747
- <button type="submit">Login</button>
748
- </form>
749
- """
750
- if request.method == "GET":
751
- return Response(form_block, status=200)
752
-
753
- if "username[$ne]" in request.form.keys() and "password[$ne]" in request.form.keys():
754
- return Response("missing username or password", status=500)
755
- if "username[$eq]" in request.form.keys() and "password[$eq]" in request.form.keys():
756
- return Response("missing username or password", status=500)
757
- else:
758
- return Response("Invalid Username or Password!", status=200)
759
-
760
- def check(self, module_test, events):
761
- finding_count = 0
762
- for e in events:
763
- if e.type == "FINDING":
764
- finding_count += 1
765
- assert finding_count == 0, "False positive FINDING emitted"
766
-
767
-
768
642
  # SQLI Single Quote/Two Single Quote (getparam)
769
643
  class Test_Lightfuzz_sqli(ModuleTestBase):
770
644
  targets = ["http://127.0.0.1:8888"]
@@ -1040,8 +914,6 @@ class Test_Lightfuzz_sqli_cookies(Test_Lightfuzz_sqli):
1040
914
  </html>
1041
915
  """
1042
916
 
1043
- print("@@@@@???")
1044
- print(request.cookies)
1045
917
  if request.cookies.get("test") is not None:
1046
918
  header_value = request.cookies.get("test")
1047
919
 
@@ -1226,7 +1098,7 @@ class Test_Lightfuzz_serial_errorresolution(ModuleTestBase):
1226
1098
  if e.type == "FINDING":
1227
1099
  if (
1228
1100
  e.data["description"]
1229
- == "POSSIBLE Unsafe Deserialization. Parameter: [TextBox1] Parameter Type: [POSTPARAM] Technique: [Error Resolution] Serialization Payload: [dotnet_base64]"
1101
+ == "POSSIBLE Unsafe Deserialization. Parameter: [TextBox1] Parameter Type: [POSTPARAM] Technique: [Error Resolution (Baseline: [500] -> Probe: [200] )] Serialization Payload: [dotnet_base64]"
1230
1102
  ):
1231
1103
  lightfuzz_serial_detect_errorresolution = True
1232
1104
 
@@ -1326,7 +1198,7 @@ class Test_Lightfuzz_serial_errorresolution_existingvalue_valid(Test_Lightfuzz_s
1326
1198
  excavate_detect_serialization_value = True
1327
1199
  if (
1328
1200
  e.data["description"]
1329
- == "POSSIBLE Unsafe Deserialization. Parameter: [TextBox1] Parameter Type: [POSTPARAM] Original Value: [AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==] Technique: [Error Resolution] Serialization Payload: [dotnet_base64]"
1201
+ == "POSSIBLE Unsafe Deserialization. Parameter: [TextBox1] Parameter Type: [POSTPARAM] Original Value: [AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==] Technique: [Error Resolution (Baseline: [500] -> Probe: [200] )] Serialization Payload: [dotnet_base64]"
1330
1202
  ):
1331
1203
  lightfuzz_serial_detect_errorresolution = True
1332
1204
 
@@ -76,7 +76,7 @@ class TestPortscan(ModuleTestBase):
76
76
  def check(self, module_test, events):
77
77
  assert set(self.syn_scanned) == {"8.8.8.0/24", "8.8.4.0/24"}
78
78
  assert set(self.ping_scanned) == set()
79
- assert self.syn_runs == 1
79
+ assert self.syn_runs >= 1
80
80
  assert self.ping_runs == 0
81
81
  assert 1 == len(
82
82
  [e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com" and str(e.module) == "TARGET"]
@@ -133,7 +133,7 @@ class TestPortscanPingFirst(TestPortscan):
133
133
  assert set(self.syn_scanned) == {"8.8.8.8/32"}
134
134
  assert set(self.ping_scanned) == {"8.8.8.0/24", "8.8.4.0/24"}
135
135
  assert self.syn_runs == 1
136
- assert self.ping_runs == 1
136
+ assert self.ping_runs >= 1
137
137
  open_port_events = [e for e in events if e.type == "OPEN_TCP_PORT"]
138
138
  assert len(open_port_events) == 3
139
139
  assert {e.data for e in open_port_events} == {"8.8.8.8:443", "evilcorp.com:443", "www.evilcorp.com:443"}
@@ -149,7 +149,7 @@ class TestPortscanPingOnly(TestPortscan):
149
149
  assert set(self.syn_scanned) == set()
150
150
  assert set(self.ping_scanned) == {"8.8.8.0/24", "8.8.4.0/24"}
151
151
  assert self.syn_runs == 0
152
- assert self.ping_runs == 1
152
+ assert self.ping_runs >= 1
153
153
  open_port_events = [e for e in events if e.type == "OPEN_TCP_PORT"]
154
154
  assert len(open_port_events) == 0
155
155
  ip_events = [e for e in events if e.type == "IP_ADDRESS"]
@@ -1,8 +1,13 @@
1
1
  from .base import ModuleTestBase
2
+ from bbot.test.bbot_fixtures import bbot_test_dir
2
3
 
3
4
 
4
5
  class TestPostman_Download(ModuleTestBase):
5
- config_overrides = {"modules": {"postman_download": {"api_key": "asdf"}}}
6
+ config_overrides = {
7
+ "modules": {
8
+ "postman_download": {"api_key": "asdf", "output_folder": str(bbot_test_dir / "test_postman_files")}
9
+ }
10
+ }
6
11
  modules_overrides = ["postman", "postman_download", "speculate"]
7
12
 
8
13
  async def setup_before_prep(self, module_test):
@@ -1,15 +1,24 @@
1
- import subprocess
2
- import shutil
3
1
  import io
2
+ import shutil
4
3
  import zipfile
5
4
  import tarfile
5
+ import subprocess
6
+ from copy import copy
6
7
  from pathlib import Path
7
8
 
8
9
  from .base import ModuleTestBase
10
+ from bbot.test.bbot_fixtures import bbot_test_dir
9
11
 
10
12
 
11
13
  class TestTrufflehog(ModuleTestBase):
12
- config_overrides = {"modules": {"postman_download": {"api_key": "asdf"}}}
14
+ download_dir = bbot_test_dir / "test_trufflehog"
15
+ config_overrides = {
16
+ "modules": {
17
+ "postman_download": {"api_key": "asdf", "output_folder": str(download_dir)},
18
+ "docker_pull": {"output_folder": str(download_dir)},
19
+ "git_clone": {"output_folder": str(download_dir)},
20
+ }
21
+ }
13
22
  modules_overrides = [
14
23
  "github_org",
15
24
  "speculate",
@@ -1129,15 +1138,20 @@ class TestTrufflehog(ModuleTestBase):
1129
1138
  cwd=temp_repo_path,
1130
1139
  )
1131
1140
 
1132
- old_filter_event = module_test.scan.modules["git_clone"].filter_event
1141
+ # we need this test to work offline, so we patch git_clone to pull from a local file:// path
1142
+ old_handle_event = module_test.scan.modules["git_clone"].handle_event
1133
1143
 
1134
- def new_filter_event(event):
1135
- event.data["url"] = event.data["url"].replace(
1136
- "https://github.com/blacklanternsecurity", f"file://{temp_path}"
1137
- )
1138
- return old_filter_event(event)
1144
+ async def new_handle_event(event):
1145
+ if event.type == "CODE_REPOSITORY":
1146
+ event = copy(event)
1147
+ data = dict(event.data)
1148
+ data["url"] = event.data["url"].replace(
1149
+ "https://github.com/blacklanternsecurity", f"file://{temp_path}"
1150
+ )
1151
+ event.data = data
1152
+ return await old_handle_event(event)
1139
1153
 
1140
- module_test.monkeypatch.setattr(module_test.scan.modules["git_clone"], "filter_event", new_filter_event)
1154
+ module_test.monkeypatch.setattr(module_test.scan.modules["git_clone"], "handle_event", new_handle_event)
1141
1155
 
1142
1156
  def check(self, module_test, events):
1143
1157
  vuln_events = [
@@ -1201,7 +1215,15 @@ class TestTrufflehog(ModuleTestBase):
1201
1215
 
1202
1216
 
1203
1217
  class TestTrufflehog_NonVerified(TestTrufflehog):
1204
- config_overrides = {"modules": {"trufflehog": {"only_verified": False}, "postman_download": {"api_key": "asdf"}}}
1218
+ download_dir = bbot_test_dir / "test_trufflehog_nonverified"
1219
+ config_overrides = {
1220
+ "modules": {
1221
+ "trufflehog": {"only_verified": False},
1222
+ "docker_pull": {"output_folder": str(download_dir)},
1223
+ "postman_download": {"api_key": "asdf", "output_folder": str(download_dir)},
1224
+ "git_clone": {"output_folder": str(download_dir)},
1225
+ }
1226
+ }
1205
1227
 
1206
1228
  def check(self, module_test, events):
1207
1229
  finding_events = [
@@ -1279,7 +1301,11 @@ class TestTrufflehog_HTTPResponse(ModuleTestBase):
1279
1301
  class TestTrufflehog_RAWText(ModuleTestBase):
1280
1302
  targets = ["http://127.0.0.1:8888/test.pdf"]
1281
1303
  modules_overrides = ["httpx", "trufflehog", "filedownload", "extractous"]
1282
- config_overrides = {"modules": {"trufflehog": {"only_verified": False}}}
1304
+
1305
+ download_dir = bbot_test_dir / "test_trufflehog_rawtext"
1306
+ config_overrides = {
1307
+ "modules": {"trufflehog": {"only_verified": False}, "filedownload": {"output_folder": str(download_dir)}}
1308
+ }
1283
1309
 
1284
1310
  async def setup_before_prep(self, module_test):
1285
1311
  expect_args = {
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: bbot
3
- Version: 2.4.2.6677rc0
3
+ Version: 2.5.0.6715rc0
4
4
  Summary: OSINT automation for hackers.
5
5
  License: GPL-3.0
6
6
  Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool