bbot 2.2.0.5242rc0__py3-none-any.whl → 2.2.0.5279rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of bbot might be problematic. Click here for more details.

Files changed (39) hide show
  1. bbot/__init__.py +1 -1
  2. bbot/core/config/logger.py +6 -3
  3. bbot/core/core.py +20 -3
  4. bbot/core/engine.py +3 -3
  5. bbot/core/event/base.py +16 -12
  6. bbot/core/helpers/command.py +4 -3
  7. bbot/core/helpers/depsinstaller/installer.py +13 -5
  8. bbot/core/helpers/misc.py +18 -0
  9. bbot/core/helpers/process.py +0 -18
  10. bbot/core/multiprocess.py +58 -0
  11. bbot/defaults.yml +3 -0
  12. bbot/modules/censys.py +9 -13
  13. bbot/modules/internal/dnsresolve.py +12 -1
  14. bbot/modules/internal/speculate.py +33 -23
  15. bbot/modules/leakix.py +2 -3
  16. bbot/modules/passivetotal.py +9 -11
  17. bbot/presets/fast.yml +16 -0
  18. bbot/scanner/preset/args.py +17 -1
  19. bbot/scanner/preset/preset.py +8 -8
  20. bbot/scanner/scanner.py +4 -1
  21. bbot/test/bbot_fixtures.py +5 -2
  22. bbot/test/conftest.py +95 -84
  23. bbot/test/fastapi_test.py +17 -0
  24. bbot/test/test_step_1/test_bbot_fastapi.py +82 -0
  25. bbot/test/test_step_1/test_cli.py +29 -0
  26. bbot/test/test_step_1/test_dns.py +36 -0
  27. bbot/test/test_step_1/test_events.py +32 -1
  28. bbot/test/test_step_1/test_modules_basic.py +0 -3
  29. bbot/test/test_step_1/test_presets.py +1 -2
  30. bbot/test/test_step_1/test_web.py +3 -0
  31. bbot/test/test_step_2/module_tests/test_module_censys.py +4 -1
  32. bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +0 -6
  33. bbot/test/test_step_2/module_tests/test_module_leakix.py +5 -1
  34. bbot/test/test_step_2/module_tests/test_module_passivetotal.py +3 -1
  35. {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/METADATA +1 -1
  36. {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/RECORD +39 -35
  37. {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/LICENSE +0 -0
  38. {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/WHEEL +0 -0
  39. {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/entry_points.txt +0 -0
@@ -91,7 +91,6 @@ class BBOTArgs:
91
91
  *self.parsed.targets,
92
92
  whitelist=self.parsed.whitelist,
93
93
  blacklist=self.parsed.blacklist,
94
- strict_scope=self.parsed.strict_scope,
95
94
  name="args_preset",
96
95
  )
97
96
 
@@ -149,6 +148,9 @@ class BBOTArgs:
149
148
  if self.parsed.force:
150
149
  args_preset.force_start = self.parsed.force
151
150
 
151
+ if self.parsed.proxy:
152
+ args_preset.core.merge_custom({"web": {"http_proxy": self.parsed.proxy}})
153
+
152
154
  if self.parsed.custom_headers:
153
155
  args_preset.core.merge_custom({"web": {"http_headers": self.parsed.custom_headers}})
154
156
 
@@ -165,6 +167,10 @@ class BBOTArgs:
165
167
  except Exception as e:
166
168
  raise BBOTArgumentError(f'Error parsing command-line config option: "{config_arg}": {e}')
167
169
 
170
+ # strict scope
171
+ if self.parsed.strict_scope:
172
+ args_preset.core.merge_custom({"scope": {"strict": True}})
173
+
168
174
  return args_preset
169
175
 
170
176
  def create_parser(self, *args, **kwargs):
@@ -265,6 +271,11 @@ class BBOTArgs:
265
271
  help="Run scan even in the case of condition violations or failed module setups",
266
272
  )
267
273
  scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt")
274
+ scan.add_argument(
275
+ "--fast-mode",
276
+ action="store_true",
277
+ help="Scan only the provided targets as fast as possible, with no extra discovery",
278
+ )
268
279
  scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan")
269
280
  scan.add_argument(
270
281
  "--current-preset",
@@ -310,6 +321,7 @@ class BBOTArgs:
310
321
 
311
322
  misc = p.add_argument_group(title="Misc")
312
323
  misc.add_argument("--version", action="store_true", help="show BBOT version and exit")
324
+ misc.add_argument("--proxy", help="Use this proxy for all HTTP requests", metavar="HTTP_PROXY")
313
325
  misc.add_argument(
314
326
  "-H",
315
327
  "--custom-headers",
@@ -359,6 +371,10 @@ class BBOTArgs:
359
371
  custom_headers_dict[k] = v
360
372
  self.parsed.custom_headers = custom_headers_dict
361
373
 
374
+ # --fast-mode
375
+ if self.parsed.fast_mode:
376
+ self.parsed.preset += ["fast"]
377
+
362
378
  def validate(self):
363
379
  # validate config options
364
380
  sentinel = object()
@@ -47,7 +47,6 @@ class Preset:
47
47
  target (Target): Target(s) of scan.
48
48
  whitelist (Target): Scan whitelist (by default this is the same as `target`).
49
49
  blacklist (Target): Scan blacklist (this takes ultimate precedence).
50
- strict_scope (bool): If True, subdomains of targets are not considered to be in-scope.
51
50
  helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc.
52
51
  output_dir (pathlib.Path): Output directory for scan.
53
52
  scan_name (str): Name of scan. Defaults to random value, e.g. "demonic_jimmy".
@@ -87,7 +86,6 @@ class Preset:
87
86
  *targets,
88
87
  whitelist=None,
89
88
  blacklist=None,
90
- strict_scope=False,
91
89
  modules=None,
92
90
  output_modules=None,
93
91
  exclude_modules=None,
@@ -117,7 +115,6 @@ class Preset:
117
115
  *targets (str): Target(s) to scan. Types supported: hostnames, IPs, CIDRs, emails, open ports.
118
116
  whitelist (list, optional): Whitelisted target(s) to scan. Defaults to the same as `targets`.
119
117
  blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty.
120
- strict_scope (bool, optional): If True, subdomains of targets are not in-scope.
121
118
  modules (list[str], optional): List of scan modules to enable for the scan. Defaults to empty list.
122
119
  output_modules (list[str], optional): List of output modules to use. Defaults to csv, human, and json.
123
120
  exclude_modules (list[str], optional): List of modules to exclude from the scan.
@@ -234,7 +231,6 @@ class Preset:
234
231
  self.module_dirs = module_dirs
235
232
 
236
233
  # target / whitelist / blacklist
237
- self.strict_scope = strict_scope
238
234
  # these are temporary receptacles until they all get .baked() together
239
235
  self._seeds = set(targets if targets else [])
240
236
  self._whitelist = set(whitelist) if whitelist else whitelist
@@ -353,7 +349,6 @@ class Preset:
353
349
  else:
354
350
  self._whitelist.update(other._whitelist)
355
351
  self._blacklist.update(other._blacklist)
356
- self.strict_scope = self.strict_scope or other.strict_scope
357
352
 
358
353
  # module dirs
359
354
  self.module_dirs = self.module_dirs.union(other.module_dirs)
@@ -537,6 +532,14 @@ class Preset:
537
532
  def web_config(self):
538
533
  return self.core.config.get("web", {})
539
534
 
535
+ @property
536
+ def scope_config(self):
537
+ return self.config.get("scope", {})
538
+
539
+ @property
540
+ def strict_scope(self):
541
+ return self.scope_config.get("strict", False)
542
+
540
543
  def apply_log_level(self, apply_core=False):
541
544
  # silent takes precedence
542
545
  if self.silent:
@@ -635,7 +638,6 @@ class Preset:
635
638
  debug=preset_dict.get("debug", False),
636
639
  silent=preset_dict.get("silent", False),
637
640
  config=preset_dict.get("config"),
638
- strict_scope=preset_dict.get("strict_scope", False),
639
641
  module_dirs=preset_dict.get("module_dirs", []),
640
642
  include=list(preset_dict.get("include", [])),
641
643
  scan_name=preset_dict.get("scan_name"),
@@ -764,8 +766,6 @@ class Preset:
764
766
  preset_dict["whitelist"] = whitelist
765
767
  if blacklist:
766
768
  preset_dict["blacklist"] = blacklist
767
- if self.strict_scope:
768
- preset_dict["strict_scope"] = True
769
769
 
770
770
  # flags + modules
771
771
  if self.require_flags:
bbot/scanner/scanner.py CHANGED
@@ -10,11 +10,11 @@ from datetime import datetime
10
10
  from collections import OrderedDict
11
11
 
12
12
  from bbot import __version__
13
-
14
13
  from bbot.core.event import make_event
15
14
  from .manager import ScanIngress, ScanEgress
16
15
  from bbot.core.helpers.misc import sha1, rand_string
17
16
  from bbot.core.helpers.names_generator import random_name
17
+ from bbot.core.multiprocess import SHARED_INTERPRETER_STATE
18
18
  from bbot.core.helpers.async_helpers import async_to_sync_gen
19
19
  from bbot.errors import BBOTError, ScanError, ValidationError
20
20
 
@@ -259,6 +259,9 @@ class Scanner:
259
259
  Creates the scan's output folder, loads its modules, and calls their .setup() methods.
260
260
  """
261
261
 
262
+ # update the master PID
263
+ SHARED_INTERPRETER_STATE.update_scan_pid()
264
+
262
265
  self.helpers.mkdir(self.home)
263
266
  if not self._prepped:
264
267
  # save scan preset
@@ -15,8 +15,8 @@ from werkzeug.wrappers import Request
15
15
  from bbot.errors import * # noqa: F401
16
16
  from bbot.core import CORE
17
17
  from bbot.scanner import Preset
18
- from bbot.core.helpers.misc import mkdir, rand_string
19
18
  from bbot.core.helpers.async_helpers import get_event_loop
19
+ from bbot.core.helpers.misc import mkdir, rand_string, get_python_constraints
20
20
 
21
21
 
22
22
  log = logging.getLogger(f"bbot.test.fixtures")
@@ -229,4 +229,7 @@ def install_all_python_deps():
229
229
  deps_pip = set()
230
230
  for module in DEFAULT_PRESET.module_loader.preloaded().values():
231
231
  deps_pip.update(set(module.get("deps", {}).get("pip", [])))
232
- subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
232
+
233
+ constraint_file = tempwordlist(get_python_constraints())
234
+
235
+ subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip))
bbot/test/conftest.py CHANGED
@@ -16,7 +16,6 @@ from bbot.core.helpers.interactsh import server_list as interactsh_servers
16
16
  # silence stdout + trace
17
17
  root_logger = logging.getLogger()
18
18
  pytest_debug_file = Path(__file__).parent.parent.parent / "pytest_debug.log"
19
- print(f"pytest_debug_file: {pytest_debug_file}")
20
19
  debug_handler = logging.FileHandler(pytest_debug_file)
21
20
  debug_handler.setLevel(logging.DEBUG)
22
21
  debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
@@ -95,9 +94,21 @@ def bbot_httpserver_ssl():
95
94
  server.clear()
96
95
 
97
96
 
98
- @pytest.fixture
99
- def non_mocked_hosts() -> list:
100
- return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
97
+ def should_mock(request):
98
+ return not request.url.host in ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
99
+
100
+
101
+ def pytest_collection_modifyitems(config, items):
102
+ # make sure all tests have the httpx_mock marker
103
+ for item in items:
104
+ item.add_marker(
105
+ pytest.mark.httpx_mock(
106
+ should_mock=should_mock,
107
+ assert_all_requests_were_expected=False,
108
+ assert_all_responses_were_requested=False,
109
+ can_send_already_matched_responses=True,
110
+ )
111
+ )
101
112
 
102
113
 
103
114
  @pytest.fixture
@@ -240,80 +251,80 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): # pragma: no
240
251
 
241
252
 
242
253
  # BELOW: debugging for frozen/hung tests
243
- # import psutil
244
- # import traceback
245
- # import inspect
246
-
247
-
248
- # def _print_detailed_info(): # pragma: no cover
249
- # """
250
- # Debugging pytests hanging
251
- # """
252
- # print("=== Detailed Thread and Process Information ===\n")
253
- # try:
254
- # print("=== Threads ===")
255
- # for thread in threading.enumerate():
256
- # print(f"Thread Name: {thread.name}")
257
- # print(f"Thread ID: {thread.ident}")
258
- # print(f"Is Alive: {thread.is_alive()}")
259
- # print(f"Daemon: {thread.daemon}")
260
-
261
- # if hasattr(thread, "_target"):
262
- # target = thread._target
263
- # if target:
264
- # qualname = (
265
- # f"{target.__module__}.{target.__qualname__}"
266
- # if hasattr(target, "__qualname__")
267
- # else str(target)
268
- # )
269
- # print(f"Target Function: {qualname}")
270
-
271
- # if hasattr(thread, "_args"):
272
- # args = thread._args
273
- # kwargs = thread._kwargs if hasattr(thread, "_kwargs") else {}
274
- # arg_spec = inspect.getfullargspec(target)
275
-
276
- # all_args = list(args) + [f"{k}={v}" for k, v in kwargs.items()]
277
-
278
- # if inspect.ismethod(target) and arg_spec.args[0] == "self":
279
- # arg_spec.args.pop(0)
280
-
281
- # named_args = list(zip(arg_spec.args, all_args))
282
- # if arg_spec.varargs:
283
- # named_args.extend((f"*{arg_spec.varargs}", arg) for arg in all_args[len(arg_spec.args) :])
284
-
285
- # print("Arguments:")
286
- # for name, value in named_args:
287
- # print(f" {name}: {value}")
288
- # else:
289
- # print("Target Function: None")
290
- # else:
291
- # print("Target Function: Unknown")
292
-
293
- # print()
294
-
295
- # print("=== Processes ===")
296
- # current_process = psutil.Process()
297
- # for child in current_process.children(recursive=True):
298
- # print(f"Process ID: {child.pid}")
299
- # print(f"Name: {child.name()}")
300
- # print(f"Status: {child.status()}")
301
- # print(f"CPU Times: {child.cpu_times()}")
302
- # print(f"Memory Info: {child.memory_info()}")
303
- # print()
304
-
305
- # print("=== Current Process ===")
306
- # print(f"Process ID: {current_process.pid}")
307
- # print(f"Name: {current_process.name()}")
308
- # print(f"Status: {current_process.status()}")
309
- # print(f"CPU Times: {current_process.cpu_times()}")
310
- # print(f"Memory Info: {current_process.memory_info()}")
311
- # print()
312
-
313
- # except Exception as e:
314
- # print(f"An error occurred: {str(e)}")
315
- # print("Traceback:")
316
- # traceback.print_exc()
254
+ import psutil
255
+ import traceback
256
+ import inspect
257
+
258
+
259
+ def _print_detailed_info(): # pragma: no cover
260
+ """
261
+ Debugging pytests hanging
262
+ """
263
+ print("=== Detailed Thread and Process Information ===\n")
264
+ try:
265
+ print("=== Threads ===")
266
+ for thread in threading.enumerate():
267
+ print(f"Thread Name: {thread.name}")
268
+ print(f"Thread ID: {thread.ident}")
269
+ print(f"Is Alive: {thread.is_alive()}")
270
+ print(f"Daemon: {thread.daemon}")
271
+
272
+ if hasattr(thread, "_target"):
273
+ target = thread._target
274
+ if target:
275
+ qualname = (
276
+ f"{target.__module__}.{target.__qualname__}"
277
+ if hasattr(target, "__qualname__")
278
+ else str(target)
279
+ )
280
+ print(f"Target Function: {qualname}")
281
+
282
+ if hasattr(thread, "_args"):
283
+ args = thread._args
284
+ kwargs = thread._kwargs if hasattr(thread, "_kwargs") else {}
285
+ arg_spec = inspect.getfullargspec(target)
286
+
287
+ all_args = list(args) + [f"{k}={v}" for k, v in kwargs.items()]
288
+
289
+ if inspect.ismethod(target) and arg_spec.args[0] == "self":
290
+ arg_spec.args.pop(0)
291
+
292
+ named_args = list(zip(arg_spec.args, all_args))
293
+ if arg_spec.varargs:
294
+ named_args.extend((f"*{arg_spec.varargs}", arg) for arg in all_args[len(arg_spec.args) :])
295
+
296
+ print("Arguments:")
297
+ for name, value in named_args:
298
+ print(f" {name}: {value}")
299
+ else:
300
+ print("Target Function: None")
301
+ else:
302
+ print("Target Function: Unknown")
303
+
304
+ print()
305
+
306
+ print("=== Processes ===")
307
+ current_process = psutil.Process()
308
+ for child in current_process.children(recursive=True):
309
+ print(f"Process ID: {child.pid}")
310
+ print(f"Name: {child.name()}")
311
+ print(f"Status: {child.status()}")
312
+ print(f"CPU Times: {child.cpu_times()}")
313
+ print(f"Memory Info: {child.memory_info()}")
314
+ print()
315
+
316
+ print("=== Current Process ===")
317
+ print(f"Process ID: {current_process.pid}")
318
+ print(f"Name: {current_process.name()}")
319
+ print(f"Status: {current_process.status()}")
320
+ print(f"CPU Times: {current_process.cpu_times()}")
321
+ print(f"Memory Info: {current_process.memory_info()}")
322
+ print()
323
+
324
+ except Exception as e:
325
+ print(f"An error occurred: {str(e)}")
326
+ print("Traceback:")
327
+ traceback.print_exc()
317
328
 
318
329
 
319
330
  @pytest.hookimpl(tryfirst=True, hookwrapper=True)
@@ -331,11 +342,11 @@ def pytest_sessionfinish(session, exitstatus):
331
342
  yield
332
343
 
333
344
  # temporarily suspend stdout capture and print detailed thread info
334
- # capmanager = session.config.pluginmanager.get_plugin("capturemanager")
335
- # if capmanager:
336
- # capmanager.suspend_global_capture(in_=True)
345
+ capmanager = session.config.pluginmanager.get_plugin("capturemanager")
346
+ if capmanager:
347
+ capmanager.suspend_global_capture(in_=True)
337
348
 
338
- # _print_detailed_info()
349
+ _print_detailed_info()
339
350
 
340
- # if capmanager:
341
- # capmanager.resume_global_capture()
351
+ if capmanager:
352
+ capmanager.resume_global_capture()
@@ -0,0 +1,17 @@
1
+ from typing import List
2
+ from bbot import Scanner
3
+ from fastapi import FastAPI, Query
4
+
5
+ app = FastAPI()
6
+
7
+
8
+ @app.get("/start")
9
+ async def start(targets: List[str] = Query(...)):
10
+ scanner = Scanner(*targets, modules=["httpx"])
11
+ events = [e async for e in scanner.async_start()]
12
+ return [e.json() for e in events]
13
+
14
+
15
+ @app.get("/ping")
16
+ async def ping():
17
+ return {"status": "ok"}
@@ -0,0 +1,82 @@
1
+ import time
2
+ import httpx
3
+ import multiprocessing
4
+ from pathlib import Path
5
+ from subprocess import Popen
6
+ from contextlib import suppress
7
+
8
+ cwd = Path(__file__).parent.parent.parent
9
+
10
+
11
+ def run_bbot_multiprocess(queue):
12
+ from bbot import Scanner
13
+
14
+ scan = Scanner("http://127.0.0.1:8888", "blacklanternsecurity.com", modules=["httpx"])
15
+ events = [e.json() for e in scan.start()]
16
+ queue.put(events)
17
+
18
+
19
+ def test_bbot_multiprocess(bbot_httpserver):
20
+
21
+ bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com")
22
+
23
+ queue = multiprocessing.Queue()
24
+ events_process = multiprocessing.Process(target=run_bbot_multiprocess, args=(queue,))
25
+ events_process.start()
26
+ events_process.join()
27
+ events = queue.get()
28
+ assert len(events) >= 3
29
+ scan_events = [e for e in events if e["type"] == "SCAN"]
30
+ assert len(scan_events) == 2
31
+ assert any([e["data"] == "test@blacklanternsecurity.com" for e in events])
32
+
33
+
34
+ def test_bbot_fastapi(bbot_httpserver):
35
+
36
+ bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com")
37
+ fastapi_process = start_fastapi_server()
38
+
39
+ try:
40
+
41
+ # wait for the server to start with a timeout of 60 seconds
42
+ start_time = time.time()
43
+ while True:
44
+ try:
45
+ response = httpx.get("http://127.0.0.1:8978/ping")
46
+ response.raise_for_status()
47
+ break
48
+ except httpx.HTTPError:
49
+ if time.time() - start_time > 60:
50
+ raise TimeoutError("Server did not start within 60 seconds.")
51
+ time.sleep(0.1)
52
+ continue
53
+
54
+ # run a scan
55
+ response = httpx.get(
56
+ "http://127.0.0.1:8978/start",
57
+ params={"targets": ["http://127.0.0.1:8888", "blacklanternsecurity.com"]},
58
+ timeout=100,
59
+ )
60
+ events = response.json()
61
+ assert len(events) >= 3
62
+ scan_events = [e for e in events if e["type"] == "SCAN"]
63
+ assert len(scan_events) == 2
64
+ assert any([e["data"] == "test@blacklanternsecurity.com" for e in events])
65
+
66
+ finally:
67
+ with suppress(Exception):
68
+ fastapi_process.terminate()
69
+
70
+
71
+ def start_fastapi_server():
72
+ import os
73
+ import sys
74
+
75
+ env = os.environ.copy()
76
+ with suppress(KeyError):
77
+ del env["BBOT_TESTING"]
78
+ python_executable = str(sys.executable)
79
+ process = Popen(
80
+ [python_executable, "-m", "uvicorn", "bbot.test.fastapi_test:app", "--port", "8978"], cwd=cwd, env=env
81
+ )
82
+ return process
@@ -626,6 +626,35 @@ config:
626
626
  stdout_preset = yaml.safe_load(captured.out)
627
627
  assert stdout_preset["config"]["web"]["http_proxy"] == "http://proxy2"
628
628
 
629
+ # --fast-mode
630
+ monkeypatch.setattr("sys.argv", ["bbot", "--current-preset"])
631
+ cli.main()
632
+ captured = capsys.readouterr()
633
+ stdout_preset = yaml.safe_load(captured.out)
634
+ assert list(stdout_preset) == ["description"]
635
+
636
+ monkeypatch.setattr("sys.argv", ["bbot", "--fast", "--current-preset"])
637
+ cli.main()
638
+ captured = capsys.readouterr()
639
+ stdout_preset = yaml.safe_load(captured.out)
640
+ stdout_preset.pop("description")
641
+ assert stdout_preset == {
642
+ "config": {
643
+ "scope": {"strict": True},
644
+ "dns": {"minimal": True},
645
+ "modules": {"speculate": {"essential_only": True}},
646
+ },
647
+ "exclude_modules": ["excavate"],
648
+ }
649
+
650
+ # --proxy
651
+ monkeypatch.setattr("sys.argv", ["bbot", "--proxy", "http://127.0.0.1:8080", "--current-preset"])
652
+ cli.main()
653
+ captured = capsys.readouterr()
654
+ stdout_preset = yaml.safe_load(captured.out)
655
+ stdout_preset.pop("description")
656
+ assert stdout_preset == {"config": {"web": {"http_proxy": "http://127.0.0.1:8080"}}}
657
+
629
658
  # cli config overrides all presets
630
659
  monkeypatch.setattr(
631
660
  "sys.argv",
@@ -631,6 +631,42 @@ def custom_lookup(query, rdtype):
631
631
  assert len(modified_wildcard_events) == 0
632
632
 
633
633
 
634
+ @pytest.mark.asyncio
635
+ async def test_wildcard_deduplication(bbot_scanner):
636
+
637
+ custom_lookup = """
638
+ def custom_lookup(query, rdtype):
639
+ if rdtype == "TXT" and query.strip(".").endswith("evilcorp.com"):
640
+ return {""}
641
+ """
642
+
643
+ mock_data = {
644
+ "evilcorp.com": {"A": ["127.0.0.1"]},
645
+ }
646
+
647
+ from bbot.modules.base import BaseModule
648
+
649
+ class DummyModule(BaseModule):
650
+ watched_events = ["DNS_NAME"]
651
+ per_domain_only = True
652
+
653
+ async def handle_event(self, event):
654
+ for i in range(30):
655
+ await self.emit_event(f"www{i}.evilcorp.com", "DNS_NAME", parent=event)
656
+
657
+ # scan without omitted event type
658
+ scan = bbot_scanner(
659
+ "evilcorp.com", config={"dns": {"minimal": False, "wildcard_ignore": []}, "omit_event_types": []}
660
+ )
661
+ await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
662
+ dummy_module = DummyModule(scan)
663
+ scan.modules["dummy_module"] = dummy_module
664
+ events = [e async for e in scan.async_start()]
665
+ dns_name_events = [e for e in events if e.type == "DNS_NAME"]
666
+ assert len(dns_name_events) == 2
667
+ assert 1 == len([e for e in dns_name_events if e.data == "_wildcard.evilcorp.com"])
668
+
669
+
634
670
  @pytest.mark.asyncio
635
671
  async def test_dns_raw_records(bbot_scanner):
636
672
 
@@ -484,7 +484,6 @@ async def test_events(events, helpers):
484
484
  json_event = db_event.json()
485
485
  assert isinstance(json_event["uuid"], str)
486
486
  assert json_event["uuid"] == str(db_event.uuid)
487
- print(f"{json_event} / {db_event.uuid} / {db_event.parent_uuid} / {scan.root_event.uuid}")
488
487
  assert json_event["parent_uuid"] == str(scan.root_event.uuid)
489
488
  assert json_event["scope_distance"] == 1
490
489
  assert json_event["data"] == "evilcorp.com:80"
@@ -966,3 +965,35 @@ def test_event_magic():
966
965
  assert event.tags == {"folder"}
967
966
 
968
967
  zip_file.unlink()
968
+
969
+
970
+ def test_event_hashing():
971
+ scan = Scanner("example.com")
972
+ url_event = scan.make_event("https://api.example.com/", "URL_UNVERIFIED", parent=scan.root_event)
973
+ host_event_1 = scan.make_event("www.example.com", "DNS_NAME", parent=url_event)
974
+ host_event_2 = scan.make_event("test.example.com", "DNS_NAME", parent=url_event)
975
+ finding_data = {"description": "Custom Yara Rule [find_string] Matched via identifier [str1]"}
976
+ finding1 = scan.make_event(finding_data, "FINDING", parent=host_event_1)
977
+ finding2 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
978
+ finding3 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
979
+
980
+ assert finding1.data == {
981
+ "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
982
+ "host": "www.example.com",
983
+ }
984
+ assert finding2.data == {
985
+ "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
986
+ "host": "test.example.com",
987
+ }
988
+ assert finding3.data == {
989
+ "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
990
+ "host": "test.example.com",
991
+ }
992
+ assert finding1.id != finding2.id
993
+ assert finding2.id == finding3.id
994
+ assert finding1.data_id != finding2.data_id
995
+ assert finding2.data_id == finding3.data_id
996
+ assert finding1.data_hash != finding2.data_hash
997
+ assert finding2.data_hash == finding3.data_hash
998
+ assert hash(finding1) != hash(finding2)
999
+ assert hash(finding2) == hash(finding3)
@@ -10,9 +10,6 @@ from bbot.modules.internal.base import BaseInternalModule
10
10
 
11
11
  @pytest.mark.asyncio
12
12
  async def test_modules_basic_checks(events, httpx_mock):
13
- for http_method in ("GET", "CONNECT", "HEAD", "POST", "PUT", "TRACE", "DEBUG", "PATCH", "DELETE", "OPTIONS"):
14
- httpx_mock.add_response(method=http_method, url=re.compile(r".*"), json={"test": "test"})
15
-
16
13
  from bbot.scanner import Scanner
17
14
 
18
15
  scan = Scanner(config={"omit_event_types": ["URL_UNVERIFIED"]})
@@ -86,7 +86,6 @@ def test_preset_yaml(clean_default_config):
86
86
  debug=False,
87
87
  silent=True,
88
88
  config={"preset_test_asdf": 1},
89
- strict_scope=False,
90
89
  )
91
90
  preset1 = preset1.bake()
92
91
  assert "evilcorp.com" in preset1.target
@@ -210,7 +209,7 @@ def test_preset_scope():
210
209
  "evilcorp.org",
211
210
  whitelist=["evilcorp.de"],
212
211
  blacklist=["test.www.evilcorp.de"],
213
- strict_scope=True,
212
+ config={"scope": {"strict": True}},
214
213
  )
215
214
 
216
215
  preset1.merge(preset3)
@@ -471,6 +471,9 @@ async def test_web_cookies(bbot_scanner, httpx_mock):
471
471
  # but that they're not sent in the response
472
472
  with pytest.raises(httpx.TimeoutException):
473
473
  r = await client2.get(url="http://www2.evilcorp.com/cookies/test")
474
+ # make sure cookies are sent
475
+ r = await client2.get(url="http://www2.evilcorp.com/cookies/test", cookies={"wats": "fdsa"})
476
+ assert r.status_code == 200
474
477
  # make sure we can manually send cookies
475
478
  httpx_mock.add_response(url="http://www2.evilcorp.com/cookies/test2", match_headers={"Cookie": "fdsa=wats"})
476
479
  r = await client2.get(url="http://www2.evilcorp.com/cookies/test2", cookies={"fdsa": "wats"})