bbot 2.2.0.5242rc0__py3-none-any.whl → 2.2.0.5279rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of bbot might be problematic. Click here for more details.
- bbot/__init__.py +1 -1
- bbot/core/config/logger.py +6 -3
- bbot/core/core.py +20 -3
- bbot/core/engine.py +3 -3
- bbot/core/event/base.py +16 -12
- bbot/core/helpers/command.py +4 -3
- bbot/core/helpers/depsinstaller/installer.py +13 -5
- bbot/core/helpers/misc.py +18 -0
- bbot/core/helpers/process.py +0 -18
- bbot/core/multiprocess.py +58 -0
- bbot/defaults.yml +3 -0
- bbot/modules/censys.py +9 -13
- bbot/modules/internal/dnsresolve.py +12 -1
- bbot/modules/internal/speculate.py +33 -23
- bbot/modules/leakix.py +2 -3
- bbot/modules/passivetotal.py +9 -11
- bbot/presets/fast.yml +16 -0
- bbot/scanner/preset/args.py +17 -1
- bbot/scanner/preset/preset.py +8 -8
- bbot/scanner/scanner.py +4 -1
- bbot/test/bbot_fixtures.py +5 -2
- bbot/test/conftest.py +95 -84
- bbot/test/fastapi_test.py +17 -0
- bbot/test/test_step_1/test_bbot_fastapi.py +82 -0
- bbot/test/test_step_1/test_cli.py +29 -0
- bbot/test/test_step_1/test_dns.py +36 -0
- bbot/test/test_step_1/test_events.py +32 -1
- bbot/test/test_step_1/test_modules_basic.py +0 -3
- bbot/test/test_step_1/test_presets.py +1 -2
- bbot/test/test_step_1/test_web.py +3 -0
- bbot/test/test_step_2/module_tests/test_module_censys.py +4 -1
- bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +0 -6
- bbot/test/test_step_2/module_tests/test_module_leakix.py +5 -1
- bbot/test/test_step_2/module_tests/test_module_passivetotal.py +3 -1
- {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/METADATA +1 -1
- {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/RECORD +39 -35
- {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/LICENSE +0 -0
- {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/WHEEL +0 -0
- {bbot-2.2.0.5242rc0.dist-info → bbot-2.2.0.5279rc0.dist-info}/entry_points.txt +0 -0
bbot/scanner/preset/args.py
CHANGED
|
@@ -91,7 +91,6 @@ class BBOTArgs:
|
|
|
91
91
|
*self.parsed.targets,
|
|
92
92
|
whitelist=self.parsed.whitelist,
|
|
93
93
|
blacklist=self.parsed.blacklist,
|
|
94
|
-
strict_scope=self.parsed.strict_scope,
|
|
95
94
|
name="args_preset",
|
|
96
95
|
)
|
|
97
96
|
|
|
@@ -149,6 +148,9 @@ class BBOTArgs:
|
|
|
149
148
|
if self.parsed.force:
|
|
150
149
|
args_preset.force_start = self.parsed.force
|
|
151
150
|
|
|
151
|
+
if self.parsed.proxy:
|
|
152
|
+
args_preset.core.merge_custom({"web": {"http_proxy": self.parsed.proxy}})
|
|
153
|
+
|
|
152
154
|
if self.parsed.custom_headers:
|
|
153
155
|
args_preset.core.merge_custom({"web": {"http_headers": self.parsed.custom_headers}})
|
|
154
156
|
|
|
@@ -165,6 +167,10 @@ class BBOTArgs:
|
|
|
165
167
|
except Exception as e:
|
|
166
168
|
raise BBOTArgumentError(f'Error parsing command-line config option: "{config_arg}": {e}')
|
|
167
169
|
|
|
170
|
+
# strict scope
|
|
171
|
+
if self.parsed.strict_scope:
|
|
172
|
+
args_preset.core.merge_custom({"scope": {"strict": True}})
|
|
173
|
+
|
|
168
174
|
return args_preset
|
|
169
175
|
|
|
170
176
|
def create_parser(self, *args, **kwargs):
|
|
@@ -265,6 +271,11 @@ class BBOTArgs:
|
|
|
265
271
|
help="Run scan even in the case of condition violations or failed module setups",
|
|
266
272
|
)
|
|
267
273
|
scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt")
|
|
274
|
+
scan.add_argument(
|
|
275
|
+
"--fast-mode",
|
|
276
|
+
action="store_true",
|
|
277
|
+
help="Scan only the provided targets as fast as possible, with no extra discovery",
|
|
278
|
+
)
|
|
268
279
|
scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan")
|
|
269
280
|
scan.add_argument(
|
|
270
281
|
"--current-preset",
|
|
@@ -310,6 +321,7 @@ class BBOTArgs:
|
|
|
310
321
|
|
|
311
322
|
misc = p.add_argument_group(title="Misc")
|
|
312
323
|
misc.add_argument("--version", action="store_true", help="show BBOT version and exit")
|
|
324
|
+
misc.add_argument("--proxy", help="Use this proxy for all HTTP requests", metavar="HTTP_PROXY")
|
|
313
325
|
misc.add_argument(
|
|
314
326
|
"-H",
|
|
315
327
|
"--custom-headers",
|
|
@@ -359,6 +371,10 @@ class BBOTArgs:
|
|
|
359
371
|
custom_headers_dict[k] = v
|
|
360
372
|
self.parsed.custom_headers = custom_headers_dict
|
|
361
373
|
|
|
374
|
+
# --fast-mode
|
|
375
|
+
if self.parsed.fast_mode:
|
|
376
|
+
self.parsed.preset += ["fast"]
|
|
377
|
+
|
|
362
378
|
def validate(self):
|
|
363
379
|
# validate config options
|
|
364
380
|
sentinel = object()
|
bbot/scanner/preset/preset.py
CHANGED
|
@@ -47,7 +47,6 @@ class Preset:
|
|
|
47
47
|
target (Target): Target(s) of scan.
|
|
48
48
|
whitelist (Target): Scan whitelist (by default this is the same as `target`).
|
|
49
49
|
blacklist (Target): Scan blacklist (this takes ultimate precedence).
|
|
50
|
-
strict_scope (bool): If True, subdomains of targets are not considered to be in-scope.
|
|
51
50
|
helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc.
|
|
52
51
|
output_dir (pathlib.Path): Output directory for scan.
|
|
53
52
|
scan_name (str): Name of scan. Defaults to random value, e.g. "demonic_jimmy".
|
|
@@ -87,7 +86,6 @@ class Preset:
|
|
|
87
86
|
*targets,
|
|
88
87
|
whitelist=None,
|
|
89
88
|
blacklist=None,
|
|
90
|
-
strict_scope=False,
|
|
91
89
|
modules=None,
|
|
92
90
|
output_modules=None,
|
|
93
91
|
exclude_modules=None,
|
|
@@ -117,7 +115,6 @@ class Preset:
|
|
|
117
115
|
*targets (str): Target(s) to scan. Types supported: hostnames, IPs, CIDRs, emails, open ports.
|
|
118
116
|
whitelist (list, optional): Whitelisted target(s) to scan. Defaults to the same as `targets`.
|
|
119
117
|
blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty.
|
|
120
|
-
strict_scope (bool, optional): If True, subdomains of targets are not in-scope.
|
|
121
118
|
modules (list[str], optional): List of scan modules to enable for the scan. Defaults to empty list.
|
|
122
119
|
output_modules (list[str], optional): List of output modules to use. Defaults to csv, human, and json.
|
|
123
120
|
exclude_modules (list[str], optional): List of modules to exclude from the scan.
|
|
@@ -234,7 +231,6 @@ class Preset:
|
|
|
234
231
|
self.module_dirs = module_dirs
|
|
235
232
|
|
|
236
233
|
# target / whitelist / blacklist
|
|
237
|
-
self.strict_scope = strict_scope
|
|
238
234
|
# these are temporary receptacles until they all get .baked() together
|
|
239
235
|
self._seeds = set(targets if targets else [])
|
|
240
236
|
self._whitelist = set(whitelist) if whitelist else whitelist
|
|
@@ -353,7 +349,6 @@ class Preset:
|
|
|
353
349
|
else:
|
|
354
350
|
self._whitelist.update(other._whitelist)
|
|
355
351
|
self._blacklist.update(other._blacklist)
|
|
356
|
-
self.strict_scope = self.strict_scope or other.strict_scope
|
|
357
352
|
|
|
358
353
|
# module dirs
|
|
359
354
|
self.module_dirs = self.module_dirs.union(other.module_dirs)
|
|
@@ -537,6 +532,14 @@ class Preset:
|
|
|
537
532
|
def web_config(self):
|
|
538
533
|
return self.core.config.get("web", {})
|
|
539
534
|
|
|
535
|
+
@property
|
|
536
|
+
def scope_config(self):
|
|
537
|
+
return self.config.get("scope", {})
|
|
538
|
+
|
|
539
|
+
@property
|
|
540
|
+
def strict_scope(self):
|
|
541
|
+
return self.scope_config.get("strict", False)
|
|
542
|
+
|
|
540
543
|
def apply_log_level(self, apply_core=False):
|
|
541
544
|
# silent takes precedence
|
|
542
545
|
if self.silent:
|
|
@@ -635,7 +638,6 @@ class Preset:
|
|
|
635
638
|
debug=preset_dict.get("debug", False),
|
|
636
639
|
silent=preset_dict.get("silent", False),
|
|
637
640
|
config=preset_dict.get("config"),
|
|
638
|
-
strict_scope=preset_dict.get("strict_scope", False),
|
|
639
641
|
module_dirs=preset_dict.get("module_dirs", []),
|
|
640
642
|
include=list(preset_dict.get("include", [])),
|
|
641
643
|
scan_name=preset_dict.get("scan_name"),
|
|
@@ -764,8 +766,6 @@ class Preset:
|
|
|
764
766
|
preset_dict["whitelist"] = whitelist
|
|
765
767
|
if blacklist:
|
|
766
768
|
preset_dict["blacklist"] = blacklist
|
|
767
|
-
if self.strict_scope:
|
|
768
|
-
preset_dict["strict_scope"] = True
|
|
769
769
|
|
|
770
770
|
# flags + modules
|
|
771
771
|
if self.require_flags:
|
bbot/scanner/scanner.py
CHANGED
|
@@ -10,11 +10,11 @@ from datetime import datetime
|
|
|
10
10
|
from collections import OrderedDict
|
|
11
11
|
|
|
12
12
|
from bbot import __version__
|
|
13
|
-
|
|
14
13
|
from bbot.core.event import make_event
|
|
15
14
|
from .manager import ScanIngress, ScanEgress
|
|
16
15
|
from bbot.core.helpers.misc import sha1, rand_string
|
|
17
16
|
from bbot.core.helpers.names_generator import random_name
|
|
17
|
+
from bbot.core.multiprocess import SHARED_INTERPRETER_STATE
|
|
18
18
|
from bbot.core.helpers.async_helpers import async_to_sync_gen
|
|
19
19
|
from bbot.errors import BBOTError, ScanError, ValidationError
|
|
20
20
|
|
|
@@ -259,6 +259,9 @@ class Scanner:
|
|
|
259
259
|
Creates the scan's output folder, loads its modules, and calls their .setup() methods.
|
|
260
260
|
"""
|
|
261
261
|
|
|
262
|
+
# update the master PID
|
|
263
|
+
SHARED_INTERPRETER_STATE.update_scan_pid()
|
|
264
|
+
|
|
262
265
|
self.helpers.mkdir(self.home)
|
|
263
266
|
if not self._prepped:
|
|
264
267
|
# save scan preset
|
bbot/test/bbot_fixtures.py
CHANGED
|
@@ -15,8 +15,8 @@ from werkzeug.wrappers import Request
|
|
|
15
15
|
from bbot.errors import * # noqa: F401
|
|
16
16
|
from bbot.core import CORE
|
|
17
17
|
from bbot.scanner import Preset
|
|
18
|
-
from bbot.core.helpers.misc import mkdir, rand_string
|
|
19
18
|
from bbot.core.helpers.async_helpers import get_event_loop
|
|
19
|
+
from bbot.core.helpers.misc import mkdir, rand_string, get_python_constraints
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
log = logging.getLogger(f"bbot.test.fixtures")
|
|
@@ -229,4 +229,7 @@ def install_all_python_deps():
|
|
|
229
229
|
deps_pip = set()
|
|
230
230
|
for module in DEFAULT_PRESET.module_loader.preloaded().values():
|
|
231
231
|
deps_pip.update(set(module.get("deps", {}).get("pip", [])))
|
|
232
|
-
|
|
232
|
+
|
|
233
|
+
constraint_file = tempwordlist(get_python_constraints())
|
|
234
|
+
|
|
235
|
+
subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip))
|
bbot/test/conftest.py
CHANGED
|
@@ -16,7 +16,6 @@ from bbot.core.helpers.interactsh import server_list as interactsh_servers
|
|
|
16
16
|
# silence stdout + trace
|
|
17
17
|
root_logger = logging.getLogger()
|
|
18
18
|
pytest_debug_file = Path(__file__).parent.parent.parent / "pytest_debug.log"
|
|
19
|
-
print(f"pytest_debug_file: {pytest_debug_file}")
|
|
20
19
|
debug_handler = logging.FileHandler(pytest_debug_file)
|
|
21
20
|
debug_handler.setLevel(logging.DEBUG)
|
|
22
21
|
debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
|
|
@@ -95,9 +94,21 @@ def bbot_httpserver_ssl():
|
|
|
95
94
|
server.clear()
|
|
96
95
|
|
|
97
96
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
97
|
+
def should_mock(request):
|
|
98
|
+
return not request.url.host in ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def pytest_collection_modifyitems(config, items):
|
|
102
|
+
# make sure all tests have the httpx_mock marker
|
|
103
|
+
for item in items:
|
|
104
|
+
item.add_marker(
|
|
105
|
+
pytest.mark.httpx_mock(
|
|
106
|
+
should_mock=should_mock,
|
|
107
|
+
assert_all_requests_were_expected=False,
|
|
108
|
+
assert_all_responses_were_requested=False,
|
|
109
|
+
can_send_already_matched_responses=True,
|
|
110
|
+
)
|
|
111
|
+
)
|
|
101
112
|
|
|
102
113
|
|
|
103
114
|
@pytest.fixture
|
|
@@ -240,80 +251,80 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): # pragma: no
|
|
|
240
251
|
|
|
241
252
|
|
|
242
253
|
# BELOW: debugging for frozen/hung tests
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
254
|
+
import psutil
|
|
255
|
+
import traceback
|
|
256
|
+
import inspect
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _print_detailed_info(): # pragma: no cover
|
|
260
|
+
"""
|
|
261
|
+
Debugging pytests hanging
|
|
262
|
+
"""
|
|
263
|
+
print("=== Detailed Thread and Process Information ===\n")
|
|
264
|
+
try:
|
|
265
|
+
print("=== Threads ===")
|
|
266
|
+
for thread in threading.enumerate():
|
|
267
|
+
print(f"Thread Name: {thread.name}")
|
|
268
|
+
print(f"Thread ID: {thread.ident}")
|
|
269
|
+
print(f"Is Alive: {thread.is_alive()}")
|
|
270
|
+
print(f"Daemon: {thread.daemon}")
|
|
271
|
+
|
|
272
|
+
if hasattr(thread, "_target"):
|
|
273
|
+
target = thread._target
|
|
274
|
+
if target:
|
|
275
|
+
qualname = (
|
|
276
|
+
f"{target.__module__}.{target.__qualname__}"
|
|
277
|
+
if hasattr(target, "__qualname__")
|
|
278
|
+
else str(target)
|
|
279
|
+
)
|
|
280
|
+
print(f"Target Function: {qualname}")
|
|
281
|
+
|
|
282
|
+
if hasattr(thread, "_args"):
|
|
283
|
+
args = thread._args
|
|
284
|
+
kwargs = thread._kwargs if hasattr(thread, "_kwargs") else {}
|
|
285
|
+
arg_spec = inspect.getfullargspec(target)
|
|
286
|
+
|
|
287
|
+
all_args = list(args) + [f"{k}={v}" for k, v in kwargs.items()]
|
|
288
|
+
|
|
289
|
+
if inspect.ismethod(target) and arg_spec.args[0] == "self":
|
|
290
|
+
arg_spec.args.pop(0)
|
|
291
|
+
|
|
292
|
+
named_args = list(zip(arg_spec.args, all_args))
|
|
293
|
+
if arg_spec.varargs:
|
|
294
|
+
named_args.extend((f"*{arg_spec.varargs}", arg) for arg in all_args[len(arg_spec.args) :])
|
|
295
|
+
|
|
296
|
+
print("Arguments:")
|
|
297
|
+
for name, value in named_args:
|
|
298
|
+
print(f" {name}: {value}")
|
|
299
|
+
else:
|
|
300
|
+
print("Target Function: None")
|
|
301
|
+
else:
|
|
302
|
+
print("Target Function: Unknown")
|
|
303
|
+
|
|
304
|
+
print()
|
|
305
|
+
|
|
306
|
+
print("=== Processes ===")
|
|
307
|
+
current_process = psutil.Process()
|
|
308
|
+
for child in current_process.children(recursive=True):
|
|
309
|
+
print(f"Process ID: {child.pid}")
|
|
310
|
+
print(f"Name: {child.name()}")
|
|
311
|
+
print(f"Status: {child.status()}")
|
|
312
|
+
print(f"CPU Times: {child.cpu_times()}")
|
|
313
|
+
print(f"Memory Info: {child.memory_info()}")
|
|
314
|
+
print()
|
|
315
|
+
|
|
316
|
+
print("=== Current Process ===")
|
|
317
|
+
print(f"Process ID: {current_process.pid}")
|
|
318
|
+
print(f"Name: {current_process.name()}")
|
|
319
|
+
print(f"Status: {current_process.status()}")
|
|
320
|
+
print(f"CPU Times: {current_process.cpu_times()}")
|
|
321
|
+
print(f"Memory Info: {current_process.memory_info()}")
|
|
322
|
+
print()
|
|
323
|
+
|
|
324
|
+
except Exception as e:
|
|
325
|
+
print(f"An error occurred: {str(e)}")
|
|
326
|
+
print("Traceback:")
|
|
327
|
+
traceback.print_exc()
|
|
317
328
|
|
|
318
329
|
|
|
319
330
|
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
|
|
@@ -331,11 +342,11 @@ def pytest_sessionfinish(session, exitstatus):
|
|
|
331
342
|
yield
|
|
332
343
|
|
|
333
344
|
# temporarily suspend stdout capture and print detailed thread info
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
345
|
+
capmanager = session.config.pluginmanager.get_plugin("capturemanager")
|
|
346
|
+
if capmanager:
|
|
347
|
+
capmanager.suspend_global_capture(in_=True)
|
|
337
348
|
|
|
338
|
-
|
|
349
|
+
_print_detailed_info()
|
|
339
350
|
|
|
340
|
-
|
|
341
|
-
|
|
351
|
+
if capmanager:
|
|
352
|
+
capmanager.resume_global_capture()
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from typing import List
|
|
2
|
+
from bbot import Scanner
|
|
3
|
+
from fastapi import FastAPI, Query
|
|
4
|
+
|
|
5
|
+
app = FastAPI()
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@app.get("/start")
|
|
9
|
+
async def start(targets: List[str] = Query(...)):
|
|
10
|
+
scanner = Scanner(*targets, modules=["httpx"])
|
|
11
|
+
events = [e async for e in scanner.async_start()]
|
|
12
|
+
return [e.json() for e in events]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@app.get("/ping")
|
|
16
|
+
async def ping():
|
|
17
|
+
return {"status": "ok"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import httpx
|
|
3
|
+
import multiprocessing
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from subprocess import Popen
|
|
6
|
+
from contextlib import suppress
|
|
7
|
+
|
|
8
|
+
cwd = Path(__file__).parent.parent.parent
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def run_bbot_multiprocess(queue):
|
|
12
|
+
from bbot import Scanner
|
|
13
|
+
|
|
14
|
+
scan = Scanner("http://127.0.0.1:8888", "blacklanternsecurity.com", modules=["httpx"])
|
|
15
|
+
events = [e.json() for e in scan.start()]
|
|
16
|
+
queue.put(events)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_bbot_multiprocess(bbot_httpserver):
|
|
20
|
+
|
|
21
|
+
bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com")
|
|
22
|
+
|
|
23
|
+
queue = multiprocessing.Queue()
|
|
24
|
+
events_process = multiprocessing.Process(target=run_bbot_multiprocess, args=(queue,))
|
|
25
|
+
events_process.start()
|
|
26
|
+
events_process.join()
|
|
27
|
+
events = queue.get()
|
|
28
|
+
assert len(events) >= 3
|
|
29
|
+
scan_events = [e for e in events if e["type"] == "SCAN"]
|
|
30
|
+
assert len(scan_events) == 2
|
|
31
|
+
assert any([e["data"] == "test@blacklanternsecurity.com" for e in events])
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_bbot_fastapi(bbot_httpserver):
|
|
35
|
+
|
|
36
|
+
bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com")
|
|
37
|
+
fastapi_process = start_fastapi_server()
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
|
|
41
|
+
# wait for the server to start with a timeout of 60 seconds
|
|
42
|
+
start_time = time.time()
|
|
43
|
+
while True:
|
|
44
|
+
try:
|
|
45
|
+
response = httpx.get("http://127.0.0.1:8978/ping")
|
|
46
|
+
response.raise_for_status()
|
|
47
|
+
break
|
|
48
|
+
except httpx.HTTPError:
|
|
49
|
+
if time.time() - start_time > 60:
|
|
50
|
+
raise TimeoutError("Server did not start within 60 seconds.")
|
|
51
|
+
time.sleep(0.1)
|
|
52
|
+
continue
|
|
53
|
+
|
|
54
|
+
# run a scan
|
|
55
|
+
response = httpx.get(
|
|
56
|
+
"http://127.0.0.1:8978/start",
|
|
57
|
+
params={"targets": ["http://127.0.0.1:8888", "blacklanternsecurity.com"]},
|
|
58
|
+
timeout=100,
|
|
59
|
+
)
|
|
60
|
+
events = response.json()
|
|
61
|
+
assert len(events) >= 3
|
|
62
|
+
scan_events = [e for e in events if e["type"] == "SCAN"]
|
|
63
|
+
assert len(scan_events) == 2
|
|
64
|
+
assert any([e["data"] == "test@blacklanternsecurity.com" for e in events])
|
|
65
|
+
|
|
66
|
+
finally:
|
|
67
|
+
with suppress(Exception):
|
|
68
|
+
fastapi_process.terminate()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def start_fastapi_server():
|
|
72
|
+
import os
|
|
73
|
+
import sys
|
|
74
|
+
|
|
75
|
+
env = os.environ.copy()
|
|
76
|
+
with suppress(KeyError):
|
|
77
|
+
del env["BBOT_TESTING"]
|
|
78
|
+
python_executable = str(sys.executable)
|
|
79
|
+
process = Popen(
|
|
80
|
+
[python_executable, "-m", "uvicorn", "bbot.test.fastapi_test:app", "--port", "8978"], cwd=cwd, env=env
|
|
81
|
+
)
|
|
82
|
+
return process
|
|
@@ -626,6 +626,35 @@ config:
|
|
|
626
626
|
stdout_preset = yaml.safe_load(captured.out)
|
|
627
627
|
assert stdout_preset["config"]["web"]["http_proxy"] == "http://proxy2"
|
|
628
628
|
|
|
629
|
+
# --fast-mode
|
|
630
|
+
monkeypatch.setattr("sys.argv", ["bbot", "--current-preset"])
|
|
631
|
+
cli.main()
|
|
632
|
+
captured = capsys.readouterr()
|
|
633
|
+
stdout_preset = yaml.safe_load(captured.out)
|
|
634
|
+
assert list(stdout_preset) == ["description"]
|
|
635
|
+
|
|
636
|
+
monkeypatch.setattr("sys.argv", ["bbot", "--fast", "--current-preset"])
|
|
637
|
+
cli.main()
|
|
638
|
+
captured = capsys.readouterr()
|
|
639
|
+
stdout_preset = yaml.safe_load(captured.out)
|
|
640
|
+
stdout_preset.pop("description")
|
|
641
|
+
assert stdout_preset == {
|
|
642
|
+
"config": {
|
|
643
|
+
"scope": {"strict": True},
|
|
644
|
+
"dns": {"minimal": True},
|
|
645
|
+
"modules": {"speculate": {"essential_only": True}},
|
|
646
|
+
},
|
|
647
|
+
"exclude_modules": ["excavate"],
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
# --proxy
|
|
651
|
+
monkeypatch.setattr("sys.argv", ["bbot", "--proxy", "http://127.0.0.1:8080", "--current-preset"])
|
|
652
|
+
cli.main()
|
|
653
|
+
captured = capsys.readouterr()
|
|
654
|
+
stdout_preset = yaml.safe_load(captured.out)
|
|
655
|
+
stdout_preset.pop("description")
|
|
656
|
+
assert stdout_preset == {"config": {"web": {"http_proxy": "http://127.0.0.1:8080"}}}
|
|
657
|
+
|
|
629
658
|
# cli config overrides all presets
|
|
630
659
|
monkeypatch.setattr(
|
|
631
660
|
"sys.argv",
|
|
@@ -631,6 +631,42 @@ def custom_lookup(query, rdtype):
|
|
|
631
631
|
assert len(modified_wildcard_events) == 0
|
|
632
632
|
|
|
633
633
|
|
|
634
|
+
@pytest.mark.asyncio
|
|
635
|
+
async def test_wildcard_deduplication(bbot_scanner):
|
|
636
|
+
|
|
637
|
+
custom_lookup = """
|
|
638
|
+
def custom_lookup(query, rdtype):
|
|
639
|
+
if rdtype == "TXT" and query.strip(".").endswith("evilcorp.com"):
|
|
640
|
+
return {""}
|
|
641
|
+
"""
|
|
642
|
+
|
|
643
|
+
mock_data = {
|
|
644
|
+
"evilcorp.com": {"A": ["127.0.0.1"]},
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
from bbot.modules.base import BaseModule
|
|
648
|
+
|
|
649
|
+
class DummyModule(BaseModule):
|
|
650
|
+
watched_events = ["DNS_NAME"]
|
|
651
|
+
per_domain_only = True
|
|
652
|
+
|
|
653
|
+
async def handle_event(self, event):
|
|
654
|
+
for i in range(30):
|
|
655
|
+
await self.emit_event(f"www{i}.evilcorp.com", "DNS_NAME", parent=event)
|
|
656
|
+
|
|
657
|
+
# scan without omitted event type
|
|
658
|
+
scan = bbot_scanner(
|
|
659
|
+
"evilcorp.com", config={"dns": {"minimal": False, "wildcard_ignore": []}, "omit_event_types": []}
|
|
660
|
+
)
|
|
661
|
+
await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
|
|
662
|
+
dummy_module = DummyModule(scan)
|
|
663
|
+
scan.modules["dummy_module"] = dummy_module
|
|
664
|
+
events = [e async for e in scan.async_start()]
|
|
665
|
+
dns_name_events = [e for e in events if e.type == "DNS_NAME"]
|
|
666
|
+
assert len(dns_name_events) == 2
|
|
667
|
+
assert 1 == len([e for e in dns_name_events if e.data == "_wildcard.evilcorp.com"])
|
|
668
|
+
|
|
669
|
+
|
|
634
670
|
@pytest.mark.asyncio
|
|
635
671
|
async def test_dns_raw_records(bbot_scanner):
|
|
636
672
|
|
|
@@ -484,7 +484,6 @@ async def test_events(events, helpers):
|
|
|
484
484
|
json_event = db_event.json()
|
|
485
485
|
assert isinstance(json_event["uuid"], str)
|
|
486
486
|
assert json_event["uuid"] == str(db_event.uuid)
|
|
487
|
-
print(f"{json_event} / {db_event.uuid} / {db_event.parent_uuid} / {scan.root_event.uuid}")
|
|
488
487
|
assert json_event["parent_uuid"] == str(scan.root_event.uuid)
|
|
489
488
|
assert json_event["scope_distance"] == 1
|
|
490
489
|
assert json_event["data"] == "evilcorp.com:80"
|
|
@@ -966,3 +965,35 @@ def test_event_magic():
|
|
|
966
965
|
assert event.tags == {"folder"}
|
|
967
966
|
|
|
968
967
|
zip_file.unlink()
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def test_event_hashing():
|
|
971
|
+
scan = Scanner("example.com")
|
|
972
|
+
url_event = scan.make_event("https://api.example.com/", "URL_UNVERIFIED", parent=scan.root_event)
|
|
973
|
+
host_event_1 = scan.make_event("www.example.com", "DNS_NAME", parent=url_event)
|
|
974
|
+
host_event_2 = scan.make_event("test.example.com", "DNS_NAME", parent=url_event)
|
|
975
|
+
finding_data = {"description": "Custom Yara Rule [find_string] Matched via identifier [str1]"}
|
|
976
|
+
finding1 = scan.make_event(finding_data, "FINDING", parent=host_event_1)
|
|
977
|
+
finding2 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
|
|
978
|
+
finding3 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
|
|
979
|
+
|
|
980
|
+
assert finding1.data == {
|
|
981
|
+
"description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
|
|
982
|
+
"host": "www.example.com",
|
|
983
|
+
}
|
|
984
|
+
assert finding2.data == {
|
|
985
|
+
"description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
|
|
986
|
+
"host": "test.example.com",
|
|
987
|
+
}
|
|
988
|
+
assert finding3.data == {
|
|
989
|
+
"description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
|
|
990
|
+
"host": "test.example.com",
|
|
991
|
+
}
|
|
992
|
+
assert finding1.id != finding2.id
|
|
993
|
+
assert finding2.id == finding3.id
|
|
994
|
+
assert finding1.data_id != finding2.data_id
|
|
995
|
+
assert finding2.data_id == finding3.data_id
|
|
996
|
+
assert finding1.data_hash != finding2.data_hash
|
|
997
|
+
assert finding2.data_hash == finding3.data_hash
|
|
998
|
+
assert hash(finding1) != hash(finding2)
|
|
999
|
+
assert hash(finding2) == hash(finding3)
|
|
@@ -10,9 +10,6 @@ from bbot.modules.internal.base import BaseInternalModule
|
|
|
10
10
|
|
|
11
11
|
@pytest.mark.asyncio
|
|
12
12
|
async def test_modules_basic_checks(events, httpx_mock):
|
|
13
|
-
for http_method in ("GET", "CONNECT", "HEAD", "POST", "PUT", "TRACE", "DEBUG", "PATCH", "DELETE", "OPTIONS"):
|
|
14
|
-
httpx_mock.add_response(method=http_method, url=re.compile(r".*"), json={"test": "test"})
|
|
15
|
-
|
|
16
13
|
from bbot.scanner import Scanner
|
|
17
14
|
|
|
18
15
|
scan = Scanner(config={"omit_event_types": ["URL_UNVERIFIED"]})
|
|
@@ -86,7 +86,6 @@ def test_preset_yaml(clean_default_config):
|
|
|
86
86
|
debug=False,
|
|
87
87
|
silent=True,
|
|
88
88
|
config={"preset_test_asdf": 1},
|
|
89
|
-
strict_scope=False,
|
|
90
89
|
)
|
|
91
90
|
preset1 = preset1.bake()
|
|
92
91
|
assert "evilcorp.com" in preset1.target
|
|
@@ -210,7 +209,7 @@ def test_preset_scope():
|
|
|
210
209
|
"evilcorp.org",
|
|
211
210
|
whitelist=["evilcorp.de"],
|
|
212
211
|
blacklist=["test.www.evilcorp.de"],
|
|
213
|
-
|
|
212
|
+
config={"scope": {"strict": True}},
|
|
214
213
|
)
|
|
215
214
|
|
|
216
215
|
preset1.merge(preset3)
|
|
@@ -471,6 +471,9 @@ async def test_web_cookies(bbot_scanner, httpx_mock):
|
|
|
471
471
|
# but that they're not sent in the response
|
|
472
472
|
with pytest.raises(httpx.TimeoutException):
|
|
473
473
|
r = await client2.get(url="http://www2.evilcorp.com/cookies/test")
|
|
474
|
+
# make sure cookies are sent
|
|
475
|
+
r = await client2.get(url="http://www2.evilcorp.com/cookies/test", cookies={"wats": "fdsa"})
|
|
476
|
+
assert r.status_code == 200
|
|
474
477
|
# make sure we can manually send cookies
|
|
475
478
|
httpx_mock.add_response(url="http://www2.evilcorp.com/cookies/test2", match_headers={"Cookie": "fdsa=wats"})
|
|
476
479
|
r = await client2.get(url="http://www2.evilcorp.com/cookies/test2", cookies={"fdsa": "wats"})
|