bbot 2.4.0.6045rc0__py3-none-any.whl → 2.4.0.6067rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.4.0.6045rc"
+__version__ = "v2.4.0.6067rc"
 
 from .scanner import Scanner, Preset

bbot/cli.py

@@ -199,7 +199,7 @@ async def _main():
             if sys.stdin.isatty():
                 # warn if any targets belong directly to a cloud provider
                 if not scan.preset.strict_scope:
-                    for event in scan.target.seeds.events:
+                    for event in scan.target.seeds.event_seeds:
                         if event.type == "DNS_NAME":
                             cloudcheck_result = scan.helpers.cloudcheck(event.host)
                             if cloudcheck_result:

bbot/core/event/base.py

@@ -18,8 +18,8 @@ from pydantic import BaseModel, field_validator
 from urllib.parse import urlparse, urljoin, parse_qs
 
 
-from .helpers import *
 from bbot.errors import *
+from .helpers import EventSeed
 from bbot.core.helpers import (
     extract_words,
     is_domain,
@@ -109,18 +109,66 @@ class BaseEvent:
     # Bypass scope checking and dns resolution, distribute immediately to modules
     # This is useful for "end-of-line" events like FINDING and VULNERABILITY
     _quick_emit = False
-    # Whether this event has been retroactively marked as part of an important discovery chain
-    _graph_important = False
-    # Disables certain data validations
-    _dummy = False
     # Data validation, if data is a dictionary
     _data_validator = None
     # Whether to increment scope distance if the child and parent hosts are the same
+    # Normally we don't want this, since scope distance only increases if the host changes
+    # But for some events like SOCIAL media profiles, this is required to prevent spidering all of facebook.com
     _scope_distance_increment_same_host = False
     # Don't allow duplicates to occur within a parent chain
     # In other words, don't emit the event if the same one already exists in its discovery context
     _suppress_chain_dupes = False
 
+    # using __slots__ dramatically reduces memory usage in large scans
+    __slots__ = [
+        # Core identification attributes
+        "_uuid",
+        "_id",
+        "_hash",
+        "_data",
+        "_data_hash",
+        # Host-related attributes
+        "__host",
+        "_host_original",
+        "_port",
+        # Parent-related attributes
+        "_parent",
+        "_parent_id",
+        "_parent_uuid",
+        # Event metadata
+        "_type",
+        "_tags",
+        "_omit",
+        "__words",
+        "_priority",
+        "_scope_distance",
+        "_module_priority",
+        "_graph_important",
+        "_resolved_hosts",
+        "_discovery_context",
+        "_discovery_context_regex",
+        "_stats_recorded",
+        "_internal",
+        "_confidence",
+        "_dummy",
+        "_module",
+        # DNS-related attributes
+        "dns_children",
+        "raw_dns_records",
+        "dns_resolve_distance",
+        # Web-related attributes
+        "web_spider_distance",
+        "parsed_url",
+        "url_extension",
+        "num_redirects",
+        # File-related attributes
+        "_data_path",
+        # Public attributes
+        "module",
+        "scan",
+        "timestamp",
+    ]
+
     def __init__(
         self,
         data,
@@ -129,7 +177,6 @@ class BaseEvent:
         context=None,
         module=None,
         scan=None,
-        scans=None,
         tags=None,
         confidence=100,
         timestamp=None,
@@ -148,7 +195,6 @@ class BaseEvent:
             parent (BaseEvent, optional): Parent event that led to this event's discovery. Defaults to None.
             module (str, optional): Module that discovered the event. Defaults to None.
             scan (Scan, optional): BBOT Scan object. Required unless _dummy is True. Defaults to None.
-            scans (list of Scan, optional): BBOT Scan objects, used primarily when unserializing an Event from the database. Defaults to None.
             tags (list of str, optional): Descriptive tags for the event. Defaults to None.
             confidence (int, optional): Confidence level for the event, on a scale of 1-100. Defaults to 100.
             timestamp (datetime, optional): Time of event discovery. Defaults to current UTC time.
@@ -174,6 +220,7 @@ class BaseEvent:
         self._host_original = None
         self._scope_distance = None
         self._module_priority = None
+        self._graph_important = False
         self._resolved_hosts = set()
         self.dns_children = {}
         self.raw_dns_records = {}
@@ -204,12 +251,6 @@ class BaseEvent:
         self.scan = scan
         if (not self.scan) and (not self._dummy):
             raise ValidationError("Must specify scan")
-        # self.scans holds a list of scan IDs from scans that encountered this event
-        self.scans = []
-        if scans is not None:
-            self.scans = scans
-        if self.scan:
-            self.scans = list(set([self.scan.id] + self.scans))
 
         try:
             self.data = self._sanitize_data(data)
@@ -1348,7 +1389,7 @@ class EMAIL_ADDRESS(BaseEvent):
         return validators.validate_email(data)
 
     def _host(self):
-        data = str(self.data).split("@")[-1]
+        data = str(self.data).rsplit("@", 1)[-1]
         host, self._port = split_host_port(data)
         return host
 
@@ -1652,7 +1693,6 @@ def make_event(
     context=None,
     module=None,
     scan=None,
-    scans=None,
     tags=None,
     confidence=100,
     dummy=False,
@@ -1712,12 +1752,11 @@ def make_event(
         tags = [tags]
     tags = set(tags)
 
+    # if data is already an event, update it with the user's kwargs
     if is_event(data):
         event = copy(data)
         if scan is not None and not event.scan:
             event.scan = scan
-        if scans is not None and not event.scans:
-            event.scans = scans
         if module is not None:
             event.module = module
         if parent is not None:
@@ -1731,8 +1770,11 @@ def make_event(
         event_type = data.type
         return event
     else:
+        # if event_type is not provided, autodetect it
         if event_type is None:
-            event_type, data = get_event_type(data)
+            event_seed = EventSeed(data)
+            event_type = event_seed.type
+            data = event_seed.data
             if not dummy:
                 log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"')
 
@@ -1776,7 +1818,6 @@ def make_event(
             context=context,
             module=module,
             scan=scan,
-            scans=scans,
             tags=tags,
             confidence=confidence,
             _dummy=dummy,
@@ -1810,7 +1851,6 @@ def event_from_json(j, siem_friendly=False):
         event_type = j["type"]
         kwargs = {
             "event_type": event_type,
-            "scans": j.get("scans", []),
             "tags": j.get("tags", []),
             "confidence": j.get("confidence", 100),
             "context": j.get("discovery_context", None),

bbot/core/event/helpers.py

@@ -1,52 +1,238 @@
-import logging
 import ipaddress
-from contextlib import suppress
-
+import regex as re
+from functools import cached_property
 from bbot.errors import ValidationError
-from bbot.core.helpers.regexes import event_type_regexes
-from bbot.core.helpers import smart_decode, smart_encode_punycode
+from bbot.core.helpers import validators
+from bbot.core.helpers.misc import split_host_port, make_ip_type
+from bbot.core.helpers import regexes, smart_decode, smart_encode_punycode
+
+bbot_event_seeds = {}
+
+
+"""
+An "Event Seed" is a lightweight event containing only the minimum logic required to:
+    - parse input to determine the event type + data
+    - validate+sanitize the data
+    - extract the host for scope purposes
 
+It's useful for quickly parsing target lists without the cpu+memory overhead of creating full-fledged BBOT events
 
-log = logging.getLogger("bbot.core.event.helpers")
+Not every type of BBOT event needs to be represented here. Only ones that are meant to be targets.
+"""
 
 
-def get_event_type(data):
+class EventSeedRegistry(type):
+    """
+    Metaclass for EventSeed that registers all subclasses in a registry.
     """
-    Determines the type of event based on the given data.
 
-    Args:
-        data (str): The data to be used for determining the event type.
+    def __new__(mcs, name, bases, attrs):
+        global bbot_event_seeds
+        cls = super().__new__(mcs, name, bases, attrs)
+        # Don't register the base EventSeed class
+        if name != "BaseEventSeed":
+            bbot_event_seeds[cls.__name__] = cls
+        return cls
 
-    Returns:
-        str: The type of event such as "IP_ADDRESS", "IP_RANGE", or "URL_UNVERIFIED".
 
-    Raises:
-        ValidationError: If the event type could not be determined.
+def EventSeed(input):
+    input = smart_encode_punycode(smart_decode(input).strip())
+    for _, event_class in bbot_event_seeds.items():
+        if hasattr(event_class, "precheck"):
+            if event_class.precheck(input):
+                return event_class(input)
+        else:
+            for regex in event_class.regexes:
+                match = regex.match(input)
+                if match:
+                    data = event_class.handle_match(match)
+                    return event_class(data)
+    raise ValidationError(f'Unable to autodetect data type from "{input}"')
 
-    Notes:
-        - Utilizes `smart_decode_punycode` and `smart_decode` to preprocess the data.
-        - Makes use of `ipaddress` standard library to check for IP and network types.
-        - Checks against a set of predefined regular expressions stored in `event_type_regexes`.
-    """
 
-    # IP address
-    with suppress(Exception):
-        ipaddress.ip_address(data)
-        return "IP_ADDRESS", data
+class BaseEventSeed(metaclass=EventSeedRegistry):
+    regexes = []
+    _target_type = "TARGET"
+
+    __slots__ = ["data", "host", "port", "input"]
+
+    def __init__(self, data):
+        self.data, self.host, self.port = self._sanitize_and_extract_host(data)
+        self.input = self._override_input(data)
+
+    @staticmethod
+    def handle_match(match):
+        """
+        Given a regex match, returns the event data
+        """
+        return match.group(0)
+
+    def _sanitize_and_extract_host(self, data):
+        """
+        Given the event data, returns the host
+
+        Returns:
+            tuple: (data, host, port)
+        """
+        return data, None, None
+
+    def _override_input(self, input):
+        return self.data
+
+    @property
+    def type(self):
+        return self.__class__.__name__
+
+    @cached_property
+    def _hash(self):
+        return hash(self.input)
+
+    def __hash__(self):
+        return self._hash
+
+    def __eq__(self, other):
+        return hash(self) == hash(other)
+
+    def __str__(self):
+        return f"EventSeed({self.input})"
+
+    def __repr__(self):
+        return str(self)
+
+
+class IP_ADDRESS(BaseEventSeed):
+    regexes = regexes.event_type_regexes["IP_ADDRESS"]
+
+    @staticmethod
+    def precheck(data):
+        try:
+            return ipaddress.ip_address(data)
+        except ValueError:
+            return False
+
+    @staticmethod
+    def _sanitize_and_extract_host(data):
+        validated = ipaddress.ip_address(data)
+        return str(validated), validated, None
+
+
+class DNS_NAME(BaseEventSeed):
+    regexes = regexes.event_type_regexes["DNS_NAME"]
+
+    @staticmethod
+    def _sanitize_and_extract_host(data):
+        validated = validators.validate_host(data)
+        return validated, validated, None
+
+
+class IP_RANGE(BaseEventSeed):
+    regexes = regexes.event_type_regexes["IP_RANGE"]
+
+    @staticmethod
+    def precheck(data):
+        try:
+            return ipaddress.ip_network(str(data), strict=False)
+        except ValueError:
+            return False
+
+    @staticmethod
+    def _sanitize_and_extract_host(data):
+        validated = ipaddress.ip_network(str(data), strict=False)
+        return str(validated), validated, None
+
+
+class OPEN_TCP_PORT(BaseEventSeed):
+    regexes = regexes.event_type_regexes["OPEN_TCP_PORT"]
+
+    @staticmethod
+    def _sanitize_and_extract_host(data):
+        validated = validators.validate_open_port(data)
+        host, port = split_host_port(validated)
+        host = make_ip_type(host)
+        return str(validated), host, port
+
+
+class URL_UNVERIFIED(BaseEventSeed):
+    regexes = regexes.event_type_regexes["URL"]
+
+    _scheme_to_port = {
+        "https": 443,
+        "http": 80,
+    }
+
+    @staticmethod
+    def _sanitize_and_extract_host(data):
+        parsed_url = validators.clean_url(data, url_querystring_remove=False)
+        scheme = parsed_url.scheme
+        host = make_ip_type(validators.validate_host(parsed_url.hostname))
+        port = parsed_url.port
+        if port is None:
+            port = URL_UNVERIFIED._scheme_to_port.get(scheme, None)
+        return parsed_url.geturl(), host, port
+
+
+class EMAIL_ADDRESS(BaseEventSeed):
+    regexes = regexes.event_type_regexes["EMAIL_ADDRESS"]
+
+    @staticmethod
+    def _sanitize_and_extract_host(data):
+        validated = validators.validate_email(data)
+        host = validated.rsplit("@", 1)[-1]
+        host, port = split_host_port(host)
+        return validated, host, port
+
+
+class ORG_STUB(BaseEventSeed):
+    regexes = (re.compile(r"^(?:ORG|ORG_STUB):(.*)"),)
+
+    def _override_input(self, input):
+        return f"ORG_STUB:{self.data}"
+
+    @staticmethod
+    def handle_match(match):
+        return match.group(1)
+
+
+class USERNAME(BaseEventSeed):
+    regexes = (re.compile(r"^(?:USER|USERNAME):(.*)"),)
+
+    def _override_input(self, input):
+        return f"USERNAME:{self.data}"
+
+    @staticmethod
+    def handle_match(match):
+        return match.group(1)
+
+
+class FILESYSTEM(BaseEventSeed):
+    regexes = (re.compile(r"^(?:FILESYSTEM|FILE|FOLDER|DIR|PATH):(.*)"),)
+
+    def _override_input(self, input):
+        return f"FILESYSTEM:{self.data['path']}"
+
+    @staticmethod
+    def handle_match(match):
+        return {"path": match.group(1)}
+
+
+class MOBILE_APP(BaseEventSeed):
+    regexes = (re.compile(r"^(?:MOBILE_APP|APK|IPA|APP):(.*)"),)
+
+    def _override_input(self, input):
+        return f"MOBILE_APP:{self.data['url']}"
+
+    @staticmethod
+    def handle_match(match):
+        return {"url": match.group(1)}
 
-    # IP network
-    with suppress(Exception):
-        ipaddress.ip_network(data, strict=False)
-        return "IP_RANGE", data
 
-    data = smart_encode_punycode(smart_decode(data).strip())
+class BLACKLIST_REGEX(BaseEventSeed):
+    regexes = (re.compile(r"^(?:RE|REGEX):(.*)"),)
+    _target_type = "BLACKLIST"
 
-    # Strict regexes
-    for t, regexes in event_type_regexes.items():
-        for r in regexes:
-            if r.match(data):
-                if t == "URL":
-                    return "URL_UNVERIFIED", data
-                return t, data
+    def _override_input(self, input):
+        return f"REGEX:{self.data}"
 
-    raise ValidationError(f'Unable to autodetect event type from "{data}"')
+    @staticmethod
+    def handle_match(match):
+        return match.group(1)

bbot/core/helpers/helper.py

@@ -159,7 +159,7 @@ class ConfigAwareHelper:
         self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter)
 
     def make_target(self, *targets, **kwargs):
-        return BaseTarget(*targets, scan=self.scan, **kwargs)
+        return BaseTarget(*targets, **kwargs)
 
     @property
     def config(self):

bbot/core/helpers/regexes.py

@@ -46,7 +46,7 @@ _dns_name_regex_with_period = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_
 dns_name_extraction_regex = re.compile(_dns_name_regex_with_period, re.I)
 dns_name_validation_regex = re.compile(r"^" + _dns_name_regex + r"$", re.I)
 
-_email_regex = r"(?:[^\W_][\w\-\.\+']{,100})@" + _dns_name_regex
+_email_regex = r"(?:[^\W_][\w\-\.\+']{,100})@" + _dns_name_regex + r"(?::[0-9]{1,5})?"
 email_regex = re.compile(_email_regex, re.I)
 
 _ptr_regex = r"(?:[0-9]{1,3}[-_\.]){3}[0-9]{1,3}"

bbot/core/helpers/web/web.py

@@ -57,7 +57,7 @@ class WebHelper(EngineClient):
         self.ssl_verify = self.config.get("ssl_verify", False)
         engine_debug = self.config.get("engine", {}).get("debug", False)
         super().__init__(
-            server_kwargs={"config": self.config, "target": self.parent_helper.preset.target.minimal},
+            server_kwargs={"config": self.config, "target": self.parent_helper.preset.target},
             debug=engine_debug,
         )
 

bbot/modules/deadly/nuclei.py

@@ -139,7 +139,9 @@ class nuclei(BaseModule):
         return True
 
     async def handle_batch(self, *events):
-        temp_target = self.helpers.make_target(*events)
+        temp_target = self.helpers.make_target()
+        for e in events:
+            temp_target.add(e.data, e)
         nuclei_input = [str(e.data) for e in events]
         async for severity, template, tags, host, url, name, extracted_results in self.execute_nuclei(nuclei_input):
             # this is necessary because sometimes nuclei is inconsistent about the data returned in the host field

bbot/modules/internal/excavate.py

@@ -776,7 +776,7 @@ class excavate(BaseInternalModule, BaseInterceptModule):
             event_draft = self.excavate.make_event(event_data, event_type, parent=event)
             if not event_draft:
                 return None
-            url_in_scope = self.excavate.scan.in_scope(event_draft)
+            url_in_scope = self.excavate.scan.in_scope(event_draft.host_filterable)
             urls_found = kwargs.get("urls_found", None)
             if urls_found:
                 exceeds_max_links = urls_found > self.excavate.scan.web_spider_links_per_page and url_in_scope

bbot/modules/output/asset_inventory.py

@@ -77,7 +77,7 @@ class asset_inventory(CSV):
             return False, "event is internal"
         if event.type not in self.watched_events:
             return False, "event type is not in watched_events"
-        if not self.scan.in_scope(event):
+        if not self.scan.in_scope(event.host):
             return False, "event is not in scope"
         if "unresolved" in event.tags:
             return False, "event is unresolved"

bbot/modules/portscan.py

@@ -1,7 +1,7 @@
 import json
 import ipaddress
 from contextlib import suppress
-from radixtarget import RadixTarget
+from radixtarget import RadixTarget, host_size_key
 
 from bbot.modules.base import BaseModule
 
@@ -65,8 +65,6 @@ class portscan(BaseModule):
             except ValueError as e:
                 return False, f"Error parsing ports '{self.ports}': {e}"
 
-        # whether we've finished scanning our original scan targets
-        self.scanned_initial_targets = False
         # keeps track of individual scanned IPs and their open ports
         # this is necessary because we may encounter more hosts with the same IP
         # and we want to avoid scanning them again
@@ -93,14 +91,6 @@ class portscan(BaseModule):
         return True
 
     async def handle_batch(self, *events):
-        # on our first run, we automatically include all our initial scan targets
-        if not self.scanned_initial_targets:
-            self.scanned_initial_targets = True
-            events = set(events)
-            events.update(
-                {e for e in self.scan.target.seeds.events if e.type in ("DNS_NAME", "IP_ADDRESS", "IP_RANGE")}
-            )
-
         # ping scan
         if self.ping_scan:
             ping_targets, ping_correlator = await self.make_targets(events, self.ping_scanned)
@@ -160,14 +150,13 @@ class portscan(BaseModule):
         """
         correlator = RadixTarget()
         targets = set()
-        for event in sorted(events, key=lambda e: e._host_size):
+        for event in sorted(events, key=lambda e: host_size_key(e.host)):
             # skip events without host
             if not event.host:
                 continue
             ips = set()
             try:
                 # first assume it's an ip address / ip range
-                # False == it's not a hostname
                 ips.add(ipaddress.ip_network(event.host, strict=False))
             except Exception:
                 # if it's a hostname, get its IPs from resolved_hosts

bbot/scanner/manager.py

@@ -1,5 +1,6 @@
 import asyncio
 from contextlib import suppress
+from radixtarget.helpers import host_size_key
 
 from bbot.modules.base import BaseInterceptModule
 
@@ -29,7 +30,7 @@ class ScanIngress(BaseInterceptModule):
         # track incoming duplicates module-by-module (for `suppress_dupes` attribute of modules)
         self.incoming_dup_tracker = set()
 
-    async def init_events(self, events=None):
+    async def init_events(self, event_seeds=None):
         """
         Initializes events by seeding the scanner with target events and distributing them for further processing.
 
@@ -37,21 +38,31 @@ class ScanIngress(BaseInterceptModule):
             - This method populates the event queue with initial target events.
             - It also marks the Scan object as finished with initialization by setting `_finished_init` to True.
         """
-        if events is None:
-            events = self.scan.target.seeds.events
-        async with self.scan._acatch(self.init_events), self._task_counter.count(self.init_events):
-            sorted_events = sorted(events, key=lambda e: len(e.data))
-            for event in [self.scan.root_event] + sorted_events:
-                event._dummy = False
-                event.web_spider_distance = 0
-                event.scan = self.scan
-                if event.parent is None:
-                    event.parent = self.scan.root_event
-                if event.module is None:
-                    event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET")
-                if event != self.scan.root_event:
-                    event.discovery_context = f"Scan {self.scan.name} seeded with " + "{event.type}: {event.data}"
+        async with (
+            self.scan._acatch(self.init_events, unhandled_is_critical=True),
+            self._task_counter.count(self.init_events),
+        ):
+            if event_seeds is None:
+                event_seeds = self.scan.target.seeds.event_seeds
+            root_event = self.scan.root_event
+            event_seeds = sorted(event_seeds, key=lambda e: (host_size_key(str(e.host)), e.data))
+            # queue root scan event
+            await self.queue_event(root_event, {})
+            target_module = self.scan._make_dummy_module(name="TARGET", _type="TARGET")
+            # queue each target in turn
+            for event_seed in event_seeds:
+                event = self.scan.make_event(
+                    event_seed.data,
+                    event_seed.type,
+                    parent=root_event,
+                    module=target_module,
+                    context=f"Scan {self.scan.name} seeded with " + "{event.type}: {event.data}",
+                    tags=["target"],
+                )
                 self.verbose(f"Target: {event}")
+                # don't fill up the queue with too many events
+                while self.incoming_event_queue.qsize() > 100:
+                    await asyncio.sleep(0.2)
                 await self.queue_event(event, {})
             await asyncio.sleep(0.1)
             self.scan._finished_init = True
@@ -95,7 +106,10 @@ class ScanIngress(BaseInterceptModule):
                 event.add_tag("blacklisted")
 
         # main scan blacklist
-        event_blacklisted = self.scan.blacklisted(event)
+        host_filterable = getattr(event, "host_filterable", None)
+        event_blacklisted = False
+        if host_filterable:
+            event_blacklisted = self.scan.blacklisted(host_filterable)
 
         # reject all blacklisted events
         if event_blacklisted or "blacklisted" in event.tags:

bbot/scanner/preset/preset.py

@@ -483,11 +483,7 @@ class Preset(metaclass=BasePreset):
         from bbot.scanner.target import BBOTTarget
 
         baked_preset._target = BBOTTarget(
-            *list(self._seeds),
-            whitelist=self._whitelist,
-            blacklist=self._blacklist,
-            strict_scope=self.strict_scope,
-            scan=scan,
+            *list(self._seeds), whitelist=self._whitelist, blacklist=self._blacklist, strict_scope=self.strict_scope
         )
 
         # evaluate conditions