bbot 2.3.0.5546rc0__py3-none-any.whl → 2.3.1.5815rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.0.5546rc"
+__version__ = "v2.3.1.5815rc"
 
 from .scanner import Scanner, Preset

bbot/cli.py

@@ -247,7 +247,7 @@ async def _main():
                         log_to_stderr(f"Error in keyboard listen task: {e}", level="ERROR")
                         log_to_stderr(traceback.format_exc(), level="TRACE")
 
-                asyncio.create_task(akeyboard_listen())
+                keyboard_listen_task = asyncio.create_task(akeyboard_listen())  # noqa F841
 
             await scan.async_start_without_generator()
 

bbot/core/engine.py

@@ -79,7 +79,7 @@ class EngineBase:
                 self.log.debug(f"{self.name}: Timeout after {interval:,} seconds {context}, retrying...")
                 retries += 1
                 if max_retries is not None and retries > max_retries:
-                    raise TimeoutError(f"Timed out after {(max_retries+1)*interval:,} seconds {context}")
+                    raise TimeoutError(f"Timed out after {(max_retries + 1) * interval:,} seconds {context}")
 
     def engine_debug(self, *args, **kwargs):
         if self._engine_debug:

bbot/core/event/base.py

@@ -1032,8 +1032,8 @@ class ClosestHostEvent(DictHostEvent):
                     self.data = self.data
                     break
         # die if we still haven't found a host
-        if not self.host:
-            raise ValueError("No host was found in event parents. Host must be specified!")
+        if not self.host and not self.data.get("path", ""):
+            raise ValueError(f"No host was found in event parents: {self.get_parents()}. Host must be specified!")
 
 
 class DictPathEvent(DictEvent):
@@ -1399,14 +1399,16 @@ class HTTP_RESPONSE(URL_UNVERIFIED, DictEvent):
         return set()
 
     def _pretty_string(self):
-        return f'{self.data["hash"]["header_mmh3"]}:{self.data["hash"]["body_mmh3"]}'
+        return f"{self.data['hash']['header_mmh3']}:{self.data['hash']['body_mmh3']}"
 
     @property
     def raw_response(self):
         """
         Formats the status code, headers, and body into a single string formatted as an HTTP/1.1 response.
         """
-        return f'{self.data["raw_header"]}{self.data["body"]}'
+        raw_header = self.data.get("raw_header", "")
+        body = self.data.get("body", "")
+        return f"{raw_header}{body}"
 
     @property
     def http_status(self):
@@ -1463,7 +1465,7 @@ class VULNERABILITY(ClosestHostEvent):
         _validate_severity = field_validator("severity")(validators.validate_severity)
 
     def _pretty_string(self):
-        return f'[{self.data["severity"]}] {self.data["description"]}'
+        return f"[{self.data['severity']}] {self.data['description']}"
 
 
 class FINDING(ClosestHostEvent):

bbot/core/helpers/async_helpers.py

@@ -51,12 +51,18 @@ class NamedLock:
 class TaskCounter:
     def __init__(self):
         self.tasks = {}
-        self.lock = asyncio.Lock()  # create a new lock
+        self._lock = None
 
     @property
     def value(self):
         return sum([t.n for t in self.tasks.values()])
 
+    @property
+    def lock(self):
+        if self._lock is None:
+            self._lock = asyncio.Lock()
+        return self._lock
+
     def count(self, task_name, n=1, _log=True):
         if callable(task_name):
             task_name = f"{task_name.__qualname__}()"

bbot/core/helpers/depsinstaller/installer.py

@@ -31,7 +31,6 @@ class DepsInstaller:
         "gcc": "gcc",
         "bash": "bash",
         "which": "which",
-        "unrar": "unrar-free",
         "tar": "tar",
         # debian why are you like this
         "7z": [
@@ -47,6 +46,12 @@ class DepsInstaller:
                 "become": True,
                 "when": "ansible_facts['os_family'] != 'Debian'",
             },
+            {
+                "name": "Install p7zip-plugins (Fedora)",
+                "package": {"name": ["p7zip-plugins"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['distribution'] == 'Fedora'",
+            },
         ],
     }
 
@@ -271,7 +276,7 @@ class DepsInstaller:
             command["cmd"] += f" && touch {command_status_file}"
             tasks.append(
                 {
-                    "name": f"{module}.deps_shell step {i+1}",
+                    "name": f"{module}.deps_shell step {i + 1}",
                     "ansible.builtin.shell": command,
                     "args": {"executable": "/bin/bash", "creates": str(command_status_file)},
                 }

bbot/core/helpers/diff.py

@@ -98,7 +98,9 @@ class HttpCompare:
                 baseline_1_json = baseline_1.text.split("\n")
                 baseline_2_json = baseline_2.text.split("\n")
 
-            ddiff = DeepDiff(baseline_1_json, baseline_2_json, ignore_order=True, view="tree")
+            ddiff = DeepDiff(
+                baseline_1_json, baseline_2_json, ignore_order=True, view="tree", threshold_to_diff_deeper=0
+            )
             self.ddiff_filters = []
 
             for k in ddiff.keys():
@@ -135,10 +137,10 @@ class HttpCompare:
             for header, value in list(headers.items()):
                 if header.lower() in self.baseline_ignore_headers:
                     with suppress(KeyError):
-                        log.debug(f'found ignored header "{header}" in headers_{i+1} and removed')
+                        log.debug(f'found ignored header "{header}" in headers_{i + 1} and removed')
                         del headers[header]
 
-        ddiff = DeepDiff(headers_1, headers_2, ignore_order=True, view="tree")
+        ddiff = DeepDiff(headers_1, headers_2, ignore_order=True, view="tree", threshold_to_diff_deeper=0)
 
         for k in ddiff.keys():
             for x in list(ddiff[k]):
@@ -153,7 +155,14 @@ class HttpCompare:
         if content_1 == content_2:
             return True
 
-        ddiff = DeepDiff(content_1, content_2, ignore_order=True, view="tree", exclude_paths=self.ddiff_filters)
+        ddiff = DeepDiff(
+            content_1,
+            content_2,
+            ignore_order=True,
+            view="tree",
+            exclude_paths=self.ddiff_filters,
+            threshold_to_diff_deeper=0,
+        )
 
         if len(ddiff.keys()) == 0:
             return True

bbot/core/helpers/dns/brute.py

@@ -29,11 +29,17 @@ class DNSBrute:
         self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)
         self.digit_regex = self.parent_helper.re.compile(r"\d+")
         self._resolver_file = None
-        self._dnsbrute_lock = asyncio.Lock()
+        self._dnsbrute_lock = None
 
     async def __call__(self, *args, **kwargs):
         return await self.dnsbrute(*args, **kwargs)
 
+    @property
+    def dnsbrute_lock(self):
+        if self._dnsbrute_lock is None:
+            self._dnsbrute_lock = asyncio.Lock()
+        return self._dnsbrute_lock
+
     async def dnsbrute(self, module, domain, subdomains, type=None):
         subdomains = list(subdomains)
 
@@ -119,7 +125,7 @@ class DNSBrute:
         )
         subdomains = self.gen_subdomains(subdomains, domain)
         hosts_yielded = set()
-        async with self._dnsbrute_lock:
+        async with self.dnsbrute_lock:
             async for line in module.run_process_live(*command, stderr=subprocess.DEVNULL, input=subdomains):
                 try:
                     j = json.loads(line)

bbot/core/helpers/dns/engine.py

@@ -658,7 +658,8 @@ class DNSEngine(EngineServer):
             assert self.in_tests, "Can only mock when BBOT_TESTING=True"
             if func_source is None:
                 return None
-            exec(func_source)
-            return locals()["custom_lookup"]
+            namespace = {}
+            exec(func_source, {}, namespace)
+            return namespace["custom_lookup"]
 
         self.resolver = MockResolver(mock_data, custom_lookup_fn=deserialize_function(custom_lookup_fn))

bbot/core/helpers/ratelimiter.py

@@ -26,9 +26,15 @@ class RateLimiter:
         self.log_interval = 10
         self.current_timestamp = time.time()
         self.count = 0
-        self.lock = asyncio.Lock()
+        self._lock = None
         self.last_notification = None
 
+    @property
+    def lock(self):
+        if self._lock is None:
+            self._lock = asyncio.Lock()
+        return self._lock
+
     async def __aenter__(self):
         async with self.lock:
             while True:
@@ -44,7 +50,7 @@ class RateLimiter:
                 else:
                     now = time.time()
                     if self.last_notification is None or now - self.last_notification >= self.log_interval:
-                        log.verbose(f"{self.name} rate limit threshold ({self.rate*10:.1f}/s) reached")
+                        log.verbose(f"{self.name} rate limit threshold ({self.rate * 10:.1f}/s) reached")
                         self.last_notification = now
                     # Rate limit for the current 0.1 second interval has been reached, wait until the next interval
                     await asyncio.sleep(self.current_timestamp + 0.1 - time.time())

bbot/core/helpers/regexes.py

@@ -38,9 +38,12 @@ _ip_range_regexes = (
 )
 ip_range_regexes = [re.compile(r, re.I) for r in _ip_range_regexes]
 
-# dns names with periods
+# all dns names including IP addresses and bare hostnames (e.g. "localhost")
 _dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.?)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
-dns_name_extraction_regex = re.compile(_dns_name_regex, re.I)
+# dns names with periods (e.g. "www.example.com")
+_dns_name_regex_with_period = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
+
+dns_name_extraction_regex = re.compile(_dns_name_regex_with_period, re.I)
 dns_name_validation_regex = re.compile(r"^" + _dns_name_regex + r"$", re.I)
 
 _email_regex = r"(?:[^\W_][\w\-\.\+']{,100})@" + _dns_name_regex

bbot/core/helpers/web/engine.py

@@ -152,7 +152,7 @@ class HTTPEngine(EngineServer):
                         log.verbose(
                             f"Size of response from {url} exceeds {bytes_to_human(max_size)}, file will be truncated"
                         )
-                        agen.aclose()
+                        await agen.aclose()
                         break
                     total_size += _chunk_size
                     chunks.append(chunk)

bbot/core/helpers/web/web.py

@@ -385,7 +385,7 @@ class WebHelper(EngineClient):
             cookies_str = ""
             for k, v in cookies.items():
                 cookies_str += f"{k}={v}; "
-            curl_command.append(f'{cookies_str.rstrip(" ")}')
+            curl_command.append(f"{cookies_str.rstrip(' ')}")
 
         path_override = kwargs.get("path_override", None)
         if path_override:

bbot/core/shared_deps.py

@@ -119,6 +119,20 @@ DEP_CHROMIUM = [
         "when": "ansible_facts['os_family'] == 'Debian'",
         "ignore_errors": True,
     },
+    # Because Ubuntu is a special snowflake, we have to bend over backwards to fix the chrome sandbox
+    # see https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md
+    {
+        "name": "Chown chrome_sandbox to root:root",
+        "command": {"cmd": "chown -R root:root #{BBOT_TOOLS}/chrome-linux/chrome_sandbox"},
+        "when": "ansible_facts['os_family'] == 'Debian'",
+        "become": True,
+    },
+    {
+        "name": "Chmod chrome_sandbox to 4755",
+        "command": {"cmd": "chmod -R 4755 #{BBOT_TOOLS}/chrome-linux/chrome_sandbox"},
+        "when": "ansible_facts['os_family'] == 'Debian'",
+        "become": True,
+    },
 ]
 
 DEP_MASSCAN = [

bbot/defaults.yml

@@ -176,6 +176,50 @@ url_extension_blacklist:
 # Distribute URLs with these extensions only to httpx (these are omitted from output)
 url_extension_httpx_only:
   - js
+
+# These url extensions are almost always static, so we exclude them from modules that fuzz things
+url_extension_static:
+  - pdf
+  - doc
+  - docx
+  - xls
+  - xlsx
+  - ppt
+  - pptx
+  - txt
+  - csv
+  - xml
+  - yaml
+  - ini
+  - log
+  - conf
+  - cfg
+  - env
+  - md
+  - rtf
+  - tiff
+  - bmp
+  - jpg
+  - jpeg
+  - png
+  - gif
+  - svg
+  - ico
+  - mp3
+  - wav
+  - flac
+  - mp4
+  - mov
+  - avi
+  - mkv
+  - webm
+  - zip
+  - tar
+  - gz
+  - bz2
+  - 7z
+  - rar
+  
 # Don't output these types of events (they are still distributed to modules)
 omit_event_types:
   - HTTP_RESPONSE

bbot/modules/ajaxpro.py

@@ -1,4 +1,5 @@
 import regex as re
+from urllib.parse import urlparse
 from bbot.modules.base import BaseModule
 
 
@@ -18,41 +19,67 @@ class ajaxpro(BaseModule):
     }
 
     async def handle_event(self, event):
-        if event.type == "URL":
-            if "dir" not in event.tags:
-                return False
-            for stem in ["ajax", "ajaxpro"]:
-                probe_url = f"{event.data}{stem}/whatever.ashx"
-                probe = await self.helpers.request(probe_url)
-                if probe:
-                    if probe.status_code == 200:
-                        probe_confirm = await self.helpers.request(f"{event.data}a/whatever.ashx")
-                        if probe_confirm:
-                            if probe_confirm.status_code != 200:
-                                await self.emit_event(
-                                    {
-                                        "host": str(event.host),
-                                        "url": event.data,
-                                        "description": f"Ajaxpro Detected (Version Unconfirmed) Trigger: [{probe_url}]",
-                                    },
-                                    "FINDING",
-                                    event,
-                                    context="{module} discovered Ajaxpro instance ({event.type}) at {event.data}",
-                                )
-
+        if event.type == "URL" and "dir" in event.tags:
+            await self.check_url_event(event)
         elif event.type == "HTTP_RESPONSE":
-            resp_body = event.data.get("body", None)
-            if resp_body:
-                ajaxpro_regex_result = await self.helpers.re.search(self.ajaxpro_regex, resp_body)
-                if ajaxpro_regex_result:
-                    ajax_pro_path = ajaxpro_regex_result.group(0)
-                    await self.emit_event(
-                        {
-                            "host": str(event.host),
-                            "url": event.data["url"],
-                            "description": f"Ajaxpro Detected (Version Unconfirmed) Trigger: [{ajax_pro_path}]",
-                        },
-                        "FINDING",
-                        event,
-                        context="{module} discovered Ajaxpro instance ({event.type}) at {event.data}",
-                    )
+            await self.check_http_response_event(event)
+
+    async def check_url_event(self, event):
+        for stem in ["ajax", "ajaxpro"]:
+            probe_url = f"{event.data}{stem}/whatever.ashx"
+            probe = await self.helpers.request(probe_url)
+            if probe and probe.status_code == 200:
+                confirm_url = f"{event.data}a/whatever.ashx"
+                confirm_probe = await self.helpers.request(confirm_url)
+                if confirm_probe and confirm_probe.status_code != 200:
+                    await self.emit_technology(event, probe_url)
+                    await self.confirm_exploitability(probe_url, event)
+
+    async def check_http_response_event(self, event):
+        resp_body = event.data.get("body")
+        if resp_body:
+            match = await self.helpers.re.search(self.ajaxpro_regex, resp_body)
+            if match:
+                ajaxpro_path = match.group(0)
+                await self.emit_technology(event, ajaxpro_path)
+                await self.confirm_exploitability(ajaxpro_path, event)
+
+    async def emit_technology(self, event, detection_url):
+        url = event.data if event.type == "URL" else event.data["url"]
+        await self.emit_event(
+            {
+                "host": str(event.host),
+                "url": url,
+                "technology": "ajaxpro",
+            },
+            "TECHNOLOGY",
+            event,
+            context=f"{self.meta['description']} discovered Ajaxpro instance ({event.type}) at {url} with trigger {detection_url}",
+        )
+
+    # Confirm exploitability of the detected Ajaxpro instance
+    async def confirm_exploitability(self, detection_url, event):
+        self.debug("Ajaxpro detected, attempting to confirm exploitability")
+        parsed_url = urlparse(detection_url)
+        base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+        path = parsed_url.path.rsplit("/", 1)[0]
+        full_url = f"{base_url}{path}/AjaxPro.Services.ICartService,AjaxPro.2.ashx"
+
+        # Payload and headers defined inline
+        payload = {}
+        headers = {"X-Ajaxpro-Method": "AddItem"}
+
+        probe_response = await self.helpers.request(full_url, method="POST", headers=headers, json=payload)
+        if probe_response:
+            if "AjaxPro.Services.ICartService" and "MissingMethodException" in probe_response.text:
+                await self.emit_event(
+                    {
+                        "host": str(event.host),
+                        "severity": "CRITICAL",
+                        "url": event.data if event.type == "URL" else event.data["url"],
+                        "description": f"Ajaxpro Deserialization RCE (CVE-2021-23758) Trigger: [{full_url}]",
+                    },
+                    "VULNERABILITY",
+                    event,
+                    context=f"{self.meta['description']} discovered Ajaxpro instance ({event.type}) at {detection_url}",
+                )

bbot/modules/baddns.py

@@ -87,10 +87,12 @@ class baddns(BaseModule):
                     for r in results:
                         r_dict = r.to_dict()
 
-                        if r_dict["confidence"] in ["CONFIRMED", "PROBABLE"]:
+                        confidence = r_dict["confidence"]
+
+                        if confidence in ["CONFIRMED", "PROBABLE"]:
                             data = {
                                 "severity": "MEDIUM",
-                                "description": f"{r_dict['description']}. Confidence: [{r_dict['confidence']}] Signature: [{r_dict['signature']}] Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]",
+                                "description": f"{r_dict['description']}. Confidence: [{confidence}] Signature: [{r_dict['signature']}] Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]",
                                 "host": str(event.host),
                             }
                             await self.emit_event(
@@ -101,20 +103,26 @@ class baddns(BaseModule):
                                 context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}',
                             )
 
-                        elif r_dict["confidence"] in ["UNLIKELY", "POSSIBLE"] and not self.only_high_confidence:
-                            data = {
-                                "description": f"{r_dict['description']} Confidence: [{r_dict['confidence']}] Signature: [{r_dict['signature']}] Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]",
-                                "host": str(event.host),
-                            }
-                            await self.emit_event(
-                                data,
-                                "FINDING",
-                                event,
-                                tags=[f"baddns-{module_instance.name.lower()}"],
-                                context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}',
-                            )
+                        elif confidence in ["UNLIKELY", "POSSIBLE"]:
+                            if not self.only_high_confidence:
+                                data = {
+                                    "description": f"{r_dict['description']} Confidence: [{confidence}] Signature: [{r_dict['signature']}] Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]",
+                                    "host": str(event.host),
+                                }
+                                await self.emit_event(
+                                    data,
+                                    "FINDING",
+                                    event,
+                                    tags=[f"baddns-{module_instance.name.lower()}"],
+                                    context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}',
+                                )
+                            else:
+                                self.debug(
+                                    f"Skipping low-confidence result due to only_high_confidence setting: {confidence}"
+                                )
+
                         else:
-                            self.warning(f"Got unrecognized confidence level: {r_dict['confidence']}")
+                            self.warning(f"Got unrecognized confidence level: {confidence}")
 
                         found_domains = r_dict.get("found_domains", None)
                         if found_domains:

bbot/modules/baddns_direct.py

@@ -49,7 +49,7 @@ class baddns_direct(BaseModule):
             "direct_mode": True,
         }
 
-        CNAME_direct_instance = CNAME_direct_module(event.host, **kwargs)
+        CNAME_direct_instance = CNAME_direct_module(str(event.host), **kwargs)
         if await CNAME_direct_instance.dispatch():
             results = CNAME_direct_instance.analyze()
             if results and len(results) > 0:
@@ -78,7 +78,7 @@ class baddns_direct(BaseModule):
         if event.type == "URL":
             if event.scope_distance > 0:
                 self.debug(
-                    f"Rejecting {event.host} due to not being in scope (scope distance: {str(event.scope_distance)})"
+                    f"Rejecting {event.host} due to not being in scope (scope distance: {event.scope_distance})"
                 )
                 return False
             if "cdn-cloudflare" not in event.tags:

bbot/modules/badsecrets.py

@@ -69,7 +69,7 @@ class badsecrets(BaseModule):
                     if r["type"] == "SecretFound":
                         data = {
                             "severity": r["description"]["severity"],
-                            "description": f"Known Secret Found. Secret Type: [{r['description']['secret']}] Secret: [{r['secret']}] Product Type: [{r['description']['product']}] Product: [{self.helpers.truncate_string(r['product'],2000)}] Detecting Module: [{r['detecting_module']}] Details: [{r['details']}]",
+                            "description": f"Known Secret Found. Secret Type: [{r['description']['secret']}] Secret: [{r['secret']}] Product Type: [{r['description']['product']}] Product: [{self.helpers.truncate_string(r['product'], 2000)}] Detecting Module: [{r['detecting_module']}] Details: [{r['details']}]",
                             "url": event.data["url"],
                             "host": str(event.host),
                         }
@@ -91,7 +91,7 @@ class badsecrets(BaseModule):
                             )
                         else:
                             data = {
-                                "description": f"Cryptographic Product identified. Product Type: [{r['description']['product']}] Product: [{self.helpers.truncate_string(r['product'],2000)}] Detecting Module: [{r['detecting_module']}]",
+                                "description": f"Cryptographic Product identified. Product Type: [{r['description']['product']}] Product: [{self.helpers.truncate_string(r['product'], 2000)}] Detecting Module: [{r['detecting_module']}]",
                                 "url": event.data["url"],
                                 "host": str(event.host),
                             }