bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/scripts/docs.py

@@ -124,7 +124,7 @@ def find_replace_file(file, keyword, replace):
         content = f.read()
         new_content = find_replace_markdown(content, keyword, replace)
     if new_content != content:
-        if not "BBOT_TESTING" in os.environ:
+        if "BBOT_TESTING" not in os.environ:
             with open(file, "w") as f:
                 f.write(new_content)
 

bbot/test/bbot_fixtures.py

@@ -15,11 +15,11 @@ from werkzeug.wrappers import Request
 from bbot.errors import *  # noqa: F401
 from bbot.core import CORE
 from bbot.scanner import Preset
-from bbot.core.helpers.misc import mkdir, rand_string
 from bbot.core.helpers.async_helpers import get_event_loop
+from bbot.core.helpers.misc import mkdir, rand_string, get_python_constraints
 
 
-log = logging.getLogger(f"bbot.test.fixtures")
+log = logging.getLogger("bbot.test.fixtures")
 
 
 bbot_test_dir = Path("/tmp/.bbot_test")
@@ -42,6 +42,13 @@ def tempwordlist(content):
     return filename
 
 
+def tempapkfile():
+    current_dir = Path(__file__).parent
+    with open(current_dir / "owasp_mastg.apk", "rb") as f:
+        apk_file = f.read()
+    return apk_file
+
+
 @pytest.fixture
 def clean_default_config(monkeypatch):
     clean_config = OmegaConf.merge(
@@ -222,4 +229,7 @@ def install_all_python_deps():
     deps_pip = set()
     for module in DEFAULT_PRESET.module_loader.preloaded().values():
         deps_pip.update(set(module.get("deps", {}).get("pip", [])))
-    subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
+
+    constraint_file = tempwordlist(get_python_constraints())
+
+    subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip))

bbot/test/conftest.py

@@ -13,17 +13,30 @@ from bbot.core import CORE
 from bbot.core.helpers.misc import execute_sync_or_async
 from bbot.core.helpers.interactsh import server_list as interactsh_servers
 
+# silence stdout + trace
+root_logger = logging.getLogger()
+pytest_debug_file = Path(__file__).parent.parent.parent / "pytest_debug.log"
+debug_handler = logging.FileHandler(pytest_debug_file)
+debug_handler.setLevel(logging.DEBUG)
+debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
+debug_handler.setFormatter(debug_format)
+root_logger.addHandler(debug_handler)
 
 test_config = OmegaConf.load(Path(__file__).parent / "test.conf")
-if test_config.get("debug", False):
-    os.environ["BBOT_DEBUG"] = "True"
-    logging.getLogger("bbot").setLevel(logging.DEBUG)
-    CORE.logger.log_level = logging.DEBUG
-else:
-    # silence stdout + trace
-    root_logger = logging.getLogger()
-    for h in root_logger.handlers:
-        h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE"))
+
+os.environ["BBOT_DEBUG"] = "True"
+CORE.logger.log_level = logging.DEBUG
+
+# silence all stderr output:
+stderr_handler = CORE.logger.log_handlers["stderr"]
+stderr_handler.setLevel(logging.CRITICAL)
+handlers = list(CORE.logger.listener.handlers)
+handlers.remove(stderr_handler)
+CORE.logger.listener.handlers = tuple(handlers)
+
+for h in root_logger.handlers:
+    h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE"))
+
 
 CORE.merge_default(test_config)
 
@@ -33,6 +46,13 @@ def assert_all_responses_were_requested() -> bool:
     return False
 
 
+@pytest.fixture(autouse=True)
+def silence_live_logging():
+    for handler in logging.getLogger().handlers:
+        if type(handler).__name__ == "_LiveLoggingStreamHandler":
+            handler.setLevel(logging.CRITICAL)
+
+
 @pytest.fixture
 def bbot_httpserver():
     server = HTTPServer(host="127.0.0.1", port=8888, threaded=True)
@@ -74,9 +94,21 @@ def bbot_httpserver_ssl():
     server.clear()
 
 
-@pytest.fixture
-def non_mocked_hosts() -> list:
-    return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
+def should_mock(request):
+    return request.url.host not in ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
+
+
+def pytest_collection_modifyitems(config, items):
+    # make sure all tests have the httpx_mock marker
+    for item in items:
+        item.add_marker(
+            pytest.mark.httpx_mock(
+                should_mock=should_mock,
+                assert_all_requests_were_expected=False,
+                assert_all_responses_were_requested=False,
+                can_send_already_matched_responses=True,
+            )
+        )
 
 
 @pytest.fixture
@@ -202,97 +234,97 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):  # pragma: no
     errors = len(stats.get("error", []))
     failed = stats.get("failed", [])
 
-    print("\nTest Session Summary:")
-    print(f"Total tests run: {total_tests}")
-    print(
-        f"{GREEN}Passed: {passed}{RESET}, {RED}Failed: {len(failed)}{RESET}, {YELLOW}Skipped: {skipped}{RESET}, Errors: {errors}"
+    terminalreporter.write("\nTest Session Summary:")
+    terminalreporter.write(f"\nTotal tests run: {total_tests}")
+    terminalreporter.write(
+        f"\n{GREEN}Passed: {passed}{RESET}, {RED}Failed: {len(failed)}{RESET}, {YELLOW}Skipped: {skipped}{RESET}, Errors: {errors}"
     )
 
     if failed:
-        print(f"\n{RED}Detailed failed test report:{RESET}")
+        terminalreporter.write(f"\n{RED}Detailed failed test report:{RESET}")
         for item in failed:
             test_name = item.nodeid.split("::")[-1] if "::" in item.nodeid else item.nodeid
             file_and_line = f"{item.location[0]}:{item.location[1]}"  # File path and line number
-            print(f"{BLUE}Test Name: {test_name}{RESET} {CYAN}({file_and_line}){RESET}")
-            print(f"{RED}Location: {item.nodeid} at {item.location[0]}:{item.location[1]}{RESET}")
-            print(f"{RED}Failure details:\n{item.longreprtext}{RESET}")
+            terminalreporter.write(f"\n{BLUE}Test Name: {test_name}{RESET} {CYAN}({file_and_line}){RESET}")
+            terminalreporter.write(f"\n{RED}Location: {item.nodeid} at {item.location[0]}:{item.location[1]}{RESET}")
+            terminalreporter.write(f"\n{RED}Failure details:\n{item.longreprtext}{RESET}")
 
 
 # BELOW: debugging for frozen/hung tests
-# import psutil
-# import traceback
-# import inspect
-
-
-# def _print_detailed_info():  # pragma: no cover
-#     """
-#     Debugging pytests hanging
-#     """
-#     print("=== Detailed Thread and Process Information ===\n")
-#     try:
-#         print("=== Threads ===")
-#         for thread in threading.enumerate():
-#             print(f"Thread Name: {thread.name}")
-#             print(f"Thread ID: {thread.ident}")
-#             print(f"Is Alive: {thread.is_alive()}")
-#             print(f"Daemon: {thread.daemon}")
-
-#             if hasattr(thread, "_target"):
-#                 target = thread._target
-#                 if target:
-#                     qualname = (
-#                         f"{target.__module__}.{target.__qualname__}"
-#                         if hasattr(target, "__qualname__")
-#                         else str(target)
-#                     )
-#                     print(f"Target Function: {qualname}")
-
-#                     if hasattr(thread, "_args"):
-#                         args = thread._args
-#                         kwargs = thread._kwargs if hasattr(thread, "_kwargs") else {}
-#                         arg_spec = inspect.getfullargspec(target)
-
-#                         all_args = list(args) + [f"{k}={v}" for k, v in kwargs.items()]
-
-#                         if inspect.ismethod(target) and arg_spec.args[0] == "self":
-#                             arg_spec.args.pop(0)
-
-#                         named_args = list(zip(arg_spec.args, all_args))
-#                         if arg_spec.varargs:
-#                             named_args.extend((f"*{arg_spec.varargs}", arg) for arg in all_args[len(arg_spec.args) :])
-
-#                         print("Arguments:")
-#                         for name, value in named_args:
-#                             print(f"  {name}: {value}")
-#                 else:
-#                     print("Target Function: None")
-#             else:
-#                 print("Target Function: Unknown")
-
-#             print()
-
-#         print("=== Processes ===")
-#         current_process = psutil.Process()
-#         for child in current_process.children(recursive=True):
-#             print(f"Process ID: {child.pid}")
-#             print(f"Name: {child.name()}")
-#             print(f"Status: {child.status()}")
-#             print(f"CPU Times: {child.cpu_times()}")
-#             print(f"Memory Info: {child.memory_info()}")
-#             print()
-
-#         print("=== Current Process ===")
-#         print(f"Process ID: {current_process.pid}")
-#         print(f"Name: {current_process.name()}")
-#         print(f"Status: {current_process.status()}")
-#         print(f"CPU Times: {current_process.cpu_times()}")
-#         print(f"Memory Info: {current_process.memory_info()}")
-#         print()
-
-#     except Exception as e:
-#         print(f"An error occurred: {str(e)}")
-#         print("Traceback:")
-#         traceback.print_exc()
+import psutil
+import traceback
+import inspect
+
+
+def _print_detailed_info():  # pragma: no cover
+    """
+    Debugging pytests hanging
+    """
+    print("=== Detailed Thread and Process Information ===\n")
+    try:
+        print("=== Threads ===")
+        for thread in threading.enumerate():
+            print(f"Thread Name: {thread.name}")
+            print(f"Thread ID: {thread.ident}")
+            print(f"Is Alive: {thread.is_alive()}")
+            print(f"Daemon: {thread.daemon}")
+
+            if hasattr(thread, "_target"):
+                target = thread._target
+                if target:
+                    qualname = (
+                        f"{target.__module__}.{target.__qualname__}"
+                        if hasattr(target, "__qualname__")
+                        else str(target)
+                    )
+                    print(f"Target Function: {qualname}")
+
+                    if hasattr(thread, "_args"):
+                        args = thread._args
+                        kwargs = thread._kwargs if hasattr(thread, "_kwargs") else {}
+                        arg_spec = inspect.getfullargspec(target)
+
+                        all_args = list(args) + [f"{k}={v}" for k, v in kwargs.items()]
+
+                        if inspect.ismethod(target) and arg_spec.args[0] == "self":
+                            arg_spec.args.pop(0)
+
+                        named_args = list(zip(arg_spec.args, all_args))
+                        if arg_spec.varargs:
+                            named_args.extend((f"*{arg_spec.varargs}", arg) for arg in all_args[len(arg_spec.args) :])
+
+                        print("Arguments:")
+                        for name, value in named_args:
+                            print(f"  {name}: {value}")
+                else:
+                    print("Target Function: None")
+            else:
+                print("Target Function: Unknown")
+
+            print()
+
+        print("=== Processes ===")
+        current_process = psutil.Process()
+        for child in current_process.children(recursive=True):
+            print(f"Process ID: {child.pid}")
+            print(f"Name: {child.name()}")
+            print(f"Status: {child.status()}")
+            print(f"CPU Times: {child.cpu_times()}")
+            print(f"Memory Info: {child.memory_info()}")
+            print()
+
+        print("=== Current Process ===")
+        print(f"Process ID: {current_process.pid}")
+        print(f"Name: {current_process.name()}")
+        print(f"Status: {current_process.status()}")
+        print(f"CPU Times: {current_process.cpu_times()}")
+        print(f"Memory Info: {current_process.memory_info()}")
+        print()
+
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")
+        print("Traceback:")
+        traceback.print_exc()
 
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
@@ -310,11 +342,11 @@ def pytest_sessionfinish(session, exitstatus):
     yield
 
     # temporarily suspend stdout capture and print detailed thread info
-    # capmanager = session.config.pluginmanager.get_plugin("capturemanager")
-    # if capmanager:
-    #     capmanager.suspend_global_capture(in_=True)
+    capmanager = session.config.pluginmanager.get_plugin("capturemanager")
+    if capmanager:
+        capmanager.suspend_global_capture(in_=True)
 
-    # _print_detailed_info()
+    _print_detailed_info()
 
-    # if capmanager:
-    #     capmanager.resume_global_capture()
+    if capmanager:
+        capmanager.resume_global_capture()

bbot/test/fastapi_test.py

@@ -0,0 +1,17 @@
+from typing import List
+from bbot import Scanner
+from fastapi import FastAPI, Query
+
+app = FastAPI()
+
+
+@app.get("/start")
+async def start(targets: List[str] = Query(...)):
+    scanner = Scanner(*targets, modules=["httpx"])
+    events = [e async for e in scanner.async_start()]
+    return [e.json() for e in events]
+
+
+@app.get("/ping")
+async def ping():
+    return {"status": "ok"}

bbot/test/run_tests.sh

@@ -3,14 +3,14 @@
 bbot_dir="$( realpath "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")"
 echo -e "[+] BBOT dir: $bbot_dir\n"
 
-echo "[+] Checking code formatting with black"
+echo "[+] Checking code formatting with ruff"
 echo "======================================="
-black --check "$bbot_dir" || exit 1
+ruff format "$bbot_dir" || exit 1
 echo
 
-echo "[+] Linting with flake8"
+echo "[+] Linting with ruff"
 echo "======================="
-flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403" "$bbot_dir" || exit 1
+ruff check "$bbot_dir" || exit 1
 echo
 
 if [ "${1}x" != "x" ] ; then

bbot/test/test.conf

@@ -36,6 +36,8 @@ dns:
     - example.com
     - evilcorp.com
     - one
+deps:
+  behavior: retry_failed
 engine:
   debug: true
 agent_url: ws://127.0.0.1:8765

bbot/test/test_step_1/test__module__tests.py

@@ -15,7 +15,6 @@ module_test_files = [m.name.split("test_module_")[-1].split(".")[0] for m in _mo
 
 
 def test__module__tests():
-
     preset = Preset()
 
     # make sure each module has a .py file

bbot/test/test_step_1/test_bbot_fastapi.py

@@ -0,0 +1,79 @@
+import time
+import httpx
+import multiprocessing
+from pathlib import Path
+from subprocess import Popen
+from contextlib import suppress
+
+cwd = Path(__file__).parent.parent.parent
+
+
+def run_bbot_multiprocess(queue):
+    from bbot import Scanner
+
+    scan = Scanner("http://127.0.0.1:8888", "blacklanternsecurity.com", modules=["httpx"])
+    events = [e.json() for e in scan.start()]
+    queue.put(events)
+
+
+def test_bbot_multiprocess(bbot_httpserver):
+    bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com")
+
+    queue = multiprocessing.Queue()
+    events_process = multiprocessing.Process(target=run_bbot_multiprocess, args=(queue,))
+    events_process.start()
+    events_process.join()
+    events = queue.get()
+    assert len(events) >= 3
+    scan_events = [e for e in events if e["type"] == "SCAN"]
+    assert len(scan_events) == 2
+    assert any(e["data"] == "test@blacklanternsecurity.com" for e in events)
+
+
+def test_bbot_fastapi(bbot_httpserver):
+    bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com")
+    fastapi_process = start_fastapi_server()
+
+    try:
+        # wait for the server to start with a timeout of 60 seconds
+        start_time = time.time()
+        while True:
+            try:
+                response = httpx.get("http://127.0.0.1:8978/ping")
+                response.raise_for_status()
+                break
+            except httpx.HTTPError:
+                if time.time() - start_time > 60:
+                    raise TimeoutError("Server did not start within 60 seconds.")
+                time.sleep(0.1)
+                continue
+
+        # run a scan
+        response = httpx.get(
+            "http://127.0.0.1:8978/start",
+            params={"targets": ["http://127.0.0.1:8888", "blacklanternsecurity.com"]},
+            timeout=100,
+        )
+        events = response.json()
+        assert len(events) >= 3
+        scan_events = [e for e in events if e["type"] == "SCAN"]
+        assert len(scan_events) == 2
+        assert any(e["data"] == "test@blacklanternsecurity.com" for e in events)
+
+    finally:
+        with suppress(Exception):
+            fastapi_process.terminate()
+
+
+def start_fastapi_server():
+    import os
+    import sys
+
+    env = os.environ.copy()
+    with suppress(KeyError):
+        del env["BBOT_TESTING"]
+    python_executable = str(sys.executable)
+    process = Popen(
+        [python_executable, "-m", "uvicorn", "bbot.test.fastapi_test:app", "--port", "8978"], cwd=cwd, env=env
+    )
+    return process

bbot/test/test_step_1/test_bloom_filter.py

@@ -6,7 +6,6 @@ import random
 
 @pytest.mark.asyncio
 async def test_bloom_filter():
-
     def generate_random_strings(n, length=10):
         """Generate a list of n random strings."""
         return ["".join(random.choices(string.ascii_letters + string.digits, k=length)) for _ in range(n)]
@@ -66,4 +65,6 @@ async def test_bloom_filter():
     # ensure false positives are less than .02 percent
     assert false_positive_percent < 0.02
 
+    bloom_filter.close()
+
     await scan._cleanup()