bbot 2.1.0.5004rc0__tar.gz → 2.1.0.5021rc0__tar.gz

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.1.0.5004rc0
+Version: 2.1.0.5021rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.1.0.5004rc"
+__version__ = "v2.1.0.5021rc"
 
 from .scanner import Scanner, Preset

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/core/event/base.py

@@ -25,6 +25,7 @@ from bbot.core.helpers import (
     is_domain,
     is_subdomain,
     is_ip,
+    is_ip_type,
     is_ptr,
     is_uri,
     url_depth,
@@ -352,6 +353,12 @@ class BaseEvent:
                 return 80
         return self._port
 
+    @property
+    def netloc(self):
+        if self.host and is_ip_type(self.host, network=False):
+            return make_netloc(self.host, self.port)
+        return None
+
     @property
     def host_stem(self):
         """
@@ -741,7 +748,7 @@ class BaseEvent:
         """
         j = dict()
         # type, ID, scope description
-        for i in ("type", "id", "uuid", "scope_description"):
+        for i in ("type", "id", "uuid", "scope_description", "netloc"):
             v = getattr(self, i, "")
             if v:
                 j.update({i: str(v)})
@@ -760,6 +767,8 @@ class BaseEvent:
             j["host"] = str(self.host)
             j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts)
             j["dns_children"] = {k: list(v) for k, v in self.dns_children.items()}
+        if isinstance(self.port, int):
+            j["port"] = self.port
         # web spider distance
         web_spider_distance = getattr(self, "web_spider_distance", None)
         if web_spider_distance is not None:

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/core/helpers/misc.py

@@ -621,12 +621,13 @@ def is_ip(d, version=None):
     return False
 
 
-def is_ip_type(i):
+def is_ip_type(i, network=None):
     """
     Checks if the given object is an instance of an IPv4 or IPv6 type from the ipaddress module.
 
     Args:
         i (ipaddress._BaseV4 or ipaddress._BaseV6): The IP object to check.
+        network (bool, optional): Whether to restrict the check to network types (IPv4Network or IPv6Network). Defaults to False.
 
     Returns:
         bool: True if the object is an instance of ipaddress._BaseV4 or ipaddress._BaseV6, False otherwise.
@@ -639,6 +640,12 @@ def is_ip_type(i):
         >>> is_ip_type("192.168.1.0/24")
         False
     """
+    if network is not None:
+        is_network = ipaddress._BaseNetwork in i.__class__.__mro__
+        if network:
+            return is_network
+        else:
+            return not is_network
     return ipaddress._IPAddressBase in i.__class__.__mro__
 
 
@@ -1260,7 +1267,7 @@ def gen_numbers(n, padding=2):
     return results
 
 
-def make_netloc(host, port):
+def make_netloc(host, port=None):
     """Constructs a network location string from a given host and port.
 
     Args:
@@ -1289,7 +1296,7 @@ def make_netloc(host, port):
     if is_ip(host, version=6):
         host = f"[{host}]"
     if port is None:
-        return host
+        return str(host)
     return f"{host}:{port}"
 
 

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/modules/base.py

@@ -984,8 +984,11 @@ class BaseModule:
     def _outgoing_dedup_hash(self, event):
         """
         Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True.
+
+        We take into account the `internal` attribute we don't want an internal event (which isn't distributed to output modules)
+        to inadvertently suppress a non-internal event.
         """
-        return hash((event, self.name))
+        return hash((event, self.name, event.internal, event.always_emit))
 
     def get_per_host_hash(self, event):
         """

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/modules/generic_ssrf.py

@@ -137,10 +137,10 @@ class Generic_XXE(BaseSubmodule):
         post_body = f"""<?xml version="1.0" encoding="ISO-8859-1"?>
 <!DOCTYPE foo [
 <!ELEMENT foo ANY >
-<!ENTITY % {rand_entity} SYSTEM "http://{subdomain_tag}.{self.parent_module.interactsh_domain}" >
+<!ENTITY {rand_entity} SYSTEM "http://{subdomain_tag}.{self.parent_module.interactsh_domain}" >
 ]>
 <foo>&{rand_entity};</foo>"""
-        test_url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/"
+        test_url = event.parsed_url.geturl()
         r = await self.parent_module.helpers.curl(
             url=test_url, method="POST", raw_body=post_body, headers={"Content-type": "application/xml"}
         )

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/modules/internal/dnsresolve.py

@@ -16,12 +16,6 @@ class DNSResolve(BaseInterceptModule):
         _name = "host"
         _type = "internal"
 
-        def _outgoing_dedup_hash(self, event):
-            # this exists to ensure a second, more interesting host isn't passed up
-            # because its ugly cousin spent its one dedup token before it arrived
-            # by removing those race conditions, this makes for more consistent results
-            return hash((event, self.name, event.always_emit))
-
     @property
     def module_threads(self):
         return self.dns_config.get("threads", 25)

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/modules/internal/excavate.py

@@ -876,6 +876,8 @@ class excavate(BaseInternalModule, BaseInterceptModule):
         yara_rules_combined = "\n".join(self.yara_rules_dict.values())
         try:
             self.info(f"Compiling {len(self.yara_rules_dict):,} YARA rules")
+            for rule_name, rule_content in self.yara_rules_dict.items():
+                self.debug(f"  - {rule_name}")
             self.yara_rules = yara.compile(source=yara_rules_combined)
         except yara.SyntaxError as e:
             self.debug(yara_rules_combined)

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/modules/internal/speculate.py

@@ -112,15 +112,15 @@ class speculate(BaseInternalModule):
         speculate_open_ports = self.emit_open_ports and event_in_scope_distance
 
         # URL --> OPEN_TCP_PORT
-        if event.type == "URL" or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
+        event_is_url = event.type == "URL"
+        if event_is_url or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
             # only speculate port from a URL if it wouldn't be speculated naturally from the host
             if event.host and (event.port not in self.ports or not speculate_open_ports):
                 await self.emit_event(
                     self.helpers.make_netloc(event.host, event.port),
                     "OPEN_TCP_PORT",
                     parent=event,
-                    internal=True,
-                    quick=(event.type == "URL"),
+                    internal=not event_is_url,  # if the URL is verified, the port is definitely open
                     context=f"speculated {{event.type}} from {event.type}: {{event.data}}",
                 )
 
@@ -169,7 +169,6 @@ class speculate(BaseInternalModule):
                         "OPEN_TCP_PORT",
                         parent=event,
                         internal=True,
-                        quick=True,
                         context="speculated {event.type}: {event.data}",
                     )
 

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/modules/shodan_dns.py

@@ -16,13 +16,11 @@ class shodan_dns(shodan):
 
     base_url = "https://api.shodan.io"
 
-    async def request_url(self, query):
-        url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}"
-        response = await self.api_request(url)
-        return response
+    async def handle_event(self, event):
+        await self.handle_event_paginated(event)
 
-    def parse_results(self, r, query):
-        json = r.json()
-        if json:
-            for hostname in json.get("subdomains", []):
-                yield f"{hostname}.{query}"
+    def make_url(self, query):
+        return f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}"
+
+    def parse_results(self, json, query):
+        return [f"{sub}.{query}" for sub in json.get("subdomains", [])]

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/modules/templates/subdomain_enum.py

@@ -31,6 +31,11 @@ class subdomain_enum(BaseModule):
     #   "lowest_parent": dedupe by lowest parent (lowest parent of www.api.test.evilcorp.com is api.test.evilcorp.com)
     dedup_strategy = "highest_parent"
 
+    # how many results to request per API call
+    page_size = 100
+    # arguments to pass to api_page_iter
+    api_page_iter_kwargs = {}
+
     @property
     def source_pretty_name(self):
         return f"{self.__class__.__name__} API"
@@ -61,10 +66,31 @@ class subdomain_enum(BaseModule):
                             context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
                         )
 
+    async def handle_event_paginated(self, event):
+        query = self.make_query(event)
+        async for result_batch in self.query_paginated(query):
+            for hostname in set(result_batch):
+                try:
+                    hostname = self.helpers.validators.validate_host(hostname)
+                except ValueError as e:
+                    self.verbose(e)
+                    continue
+                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
+                    await self.emit_event(
+                        hostname,
+                        "DNS_NAME",
+                        event,
+                        abort_if=self.abort_if,
+                        context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
+                    )
+
     async def request_url(self, query):
-        url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
+        url = self.make_url(query)
         return await self.api_request(url)
 
+    def make_url(self, query):
+        return f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
+
     def make_query(self, event):
         query = event.data
         parents = list(self.helpers.domain_parents(event.data))
@@ -86,11 +112,11 @@ class subdomain_enum(BaseModule):
             for hostname in json:
                 yield hostname
 
-    async def query(self, query, parse_fn=None, request_fn=None):
-        if parse_fn is None:
-            parse_fn = self.parse_results
+    async def query(self, query, request_fn=None, parse_fn=None):
         if request_fn is None:
             request_fn = self.request_url
+        if parse_fn is None:
+            parse_fn = self.parse_results
         try:
             response = await request_fn(query)
             if response is None:
@@ -113,6 +139,19 @@ class subdomain_enum(BaseModule):
         except Exception as e:
             self.info(f"Error retrieving results for {query}: {e}", trace=True)
 
+    async def query_paginated(self, query):
+        url = self.make_url(query)
+        agen = self.api_page_iter(url, page_size=self.page_size, **self.api_page_iter_kwargs)
+        try:
+            async for response in agen:
+                subdomains = self.parse_results(response, query)
+                self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
+                if not subdomains:
+                    break
+                yield subdomains
+        finally:
+            agen.aclose()
+
     async def _is_wildcard(self, query):
         rdtypes = ("A", "AAAA", "CNAME")
         if self.helpers.is_dns_name(query):

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/modules/trickest.py

@@ -28,39 +28,15 @@ class Trickest(subdomain_enum_apikey):
         return url, kwargs
 
     async def handle_event(self, event):
-        query = self.make_query(event)
-        async for result_batch in self.query(query):
-            for hostname in set(result_batch):
-                try:
-                    hostname = self.helpers.validators.validate_host(hostname)
-                except ValueError as e:
-                    self.verbose(e)
-                    continue
-                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
-                    await self.emit_event(
-                        hostname,
-                        "DNS_NAME",
-                        event,
-                        abort_if=self.abort_if,
-                        context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
-                    )
+        await self.handle_event_paginated(event)
 
-    async def query(self, query):
+    def make_url(self, query):
         url = f"{self.base_url}/view?q=hostname%20~%20%22.{self.helpers.quote(query)}%22"
         url += f"&dataset_id={self.dataset_id}"
         url += "&limit={page_size}&offset={offset}&select=hostname&orderby=hostname"
-        agen = self.api_page_iter(url, page_size=self.page_size)
-        try:
-            async for response in agen:
-                subdomains = self.parse_results(response)
-                self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
-                if not subdomains:
-                    break
-                yield subdomains
-        finally:
-            agen.aclose()
+        return url
 
-    def parse_results(self, j):
+    def parse_results(self, j, query):
         results = j.get("results", [])
         subdomains = set()
         for item in results:

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/modules/virustotal.py

@@ -15,6 +15,10 @@ class virustotal(subdomain_enum_apikey):
     options_desc = {"api_key": "VirusTotal API Key"}
 
     base_url = "https://www.virustotal.com/api/v3"
+    api_page_iter_kwargs = {"json": False, "next_key": lambda r: r.json().get("links", {}).get("next", "")}
+
+    def make_url(self, query):
+        return f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains"
 
     def prepare_api_request(self, url, kwargs):
         kwargs["headers"]["x-apikey"] = self.api_key
@@ -28,17 +32,3 @@ class virustotal(subdomain_enum_apikey):
             if match.endswith(query):
                 results.add(match)
         return results
-
-    async def query(self, query):
-        results = set()
-        url = f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains"
-        agen = self.api_page_iter(url, json=False, next_key=lambda r: r.json().get("links", {}).get("next", ""))
-        try:
-            async for response in agen:
-                r = self.parse_results(response, query)
-                if not r:
-                    break
-                results.update(r)
-        finally:
-            agen.aclose()
-        return results

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/scanner/scanner.py

@@ -130,20 +130,22 @@ class Scanner:
         else:
             self.id = f"SCAN:{sha1(rand_string(20)).hexdigest()}"
 
-        preset = kwargs.pop("preset", None)
+        custom_preset = kwargs.pop("preset", None)
         kwargs["_log"] = True
 
         from .preset import Preset
 
-        if preset is None:
-            preset = Preset(*targets, **kwargs)
-        else:
-            if not isinstance(preset, Preset):
-                raise ValidationError(f'Preset must be of type Preset, not "{type(preset).__name__}"')
-        self.preset = preset.bake(self)
+        base_preset = Preset(*targets, **kwargs)
+
+        if custom_preset is not None:
+            if not isinstance(custom_preset, Preset):
+                raise ValidationError(f'Preset must be of type Preset, not "{type(custom_preset).__name__}"')
+            base_preset.merge(custom_preset)
+
+        self.preset = base_preset.bake(self)
 
         # scan name
-        if preset.scan_name is None:
+        if self.preset.scan_name is None:
             tries = 0
             while 1:
                 if tries > 5:
@@ -158,7 +160,7 @@ class Scanner:
                     break
                 tries += 1
         else:
-            scan_name = str(preset.scan_name)
+            scan_name = str(self.preset.scan_name)
         self.name = scan_name
 
         # make sure the preset has a description
@@ -166,8 +168,8 @@ class Scanner:
             self.preset.description = self.name
 
         # scan output dir
-        if preset.output_dir is not None:
-            self.home = Path(preset.output_dir).resolve() / self.name
+        if self.preset.output_dir is not None:
+            self.home = Path(self.preset.output_dir).resolve() / self.name
         else:
             self.home = self.preset.bbot_home / "scans" / self.name
 

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/test/test_step_1/test_events.py

@@ -14,10 +14,19 @@ async def test_events(events, helpers):
     await scan._prep()
 
     assert events.ipv4.type == "IP_ADDRESS"
+    assert events.ipv4.netloc == "8.8.8.8"
+    assert events.ipv4.port is None
     assert events.ipv6.type == "IP_ADDRESS"
+    assert events.ipv6.netloc == "[2001:4860:4860::8888]"
+    assert events.ipv6.port is None
+    assert events.ipv6_open_port.netloc == "[2001:4860:4860::8888]:443"
     assert events.netv4.type == "IP_RANGE"
+    assert events.netv4.netloc is None
+    assert "netloc" not in events.netv4.json()
     assert events.netv6.type == "IP_RANGE"
     assert events.domain.type == "DNS_NAME"
+    assert events.domain.netloc == "publicapis.org"
+    assert events.domain.port is None
     assert "domain" in events.domain.tags
     assert events.subdomain.type == "DNS_NAME"
     assert "subdomain" in events.subdomain.tags
@@ -67,8 +76,14 @@ async def test_events(events, helpers):
     assert not events.netv6 in events.domain
     assert events.emoji not in events.domain
     assert events.domain not in events.emoji
-    assert "evilcorp.com" == scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
-    assert "evilcorp.com" == scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    open_port_event = scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
+    dns_event = scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    for e in (open_port_event, dns_event):
+        assert "evilcorp.com" == e
+        assert e.netloc == "evilcorp.com"
+        assert e.json()["netloc"] == "evilcorp.com"
+        assert e.port is None
+        assert "port" not in e.json()
 
     # url tests
     assert scan.make_event("http://evilcorp.com", dummy=True) == scan.make_event("http://evilcorp.com/", dummy=True)
@@ -78,8 +93,14 @@ async def test_events(events, helpers):
     assert "api.publicapis.org:443" in events.url_unverified
     assert "publicapis.org" not in events.url_unverified
     assert events.ipv4_url_unverified in events.ipv4
+    assert events.ipv4_url_unverified.netloc == "8.8.8.8:443"
+    assert events.ipv4_url_unverified.port == 443
+    assert events.ipv4_url_unverified.json()["port"] == 443
     assert events.ipv4_url_unverified in events.netv4
     assert events.ipv6_url_unverified in events.ipv6
+    assert events.ipv6_url_unverified.netloc == "[2001:4860:4860::8888]:443"
+    assert events.ipv6_url_unverified.port == 443
+    assert events.ipv6_url_unverified.json()["port"] == 443
     assert events.ipv6_url_unverified in events.netv6
     assert events.emoji not in events.url_unverified
     assert events.emoji not in events.ipv6_url_unverified
@@ -193,6 +214,8 @@ async def test_events(events, helpers):
 
     org_stub_1 = scan.make_event("STUB1", "ORG_STUB", parent=scan.root_event)
     org_stub_1.scope_distance == 1
+    assert org_stub_1.netloc == None
+    assert "netloc" not in org_stub_1.json()
     org_stub_2 = scan.make_event("STUB2", "ORG_STUB", parent=org_stub_1)
     org_stub_2.scope_distance == 2
 

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/test/test_step_1/test_helpers.py

@@ -94,6 +94,15 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     ]
     assert helpers.is_ip("127.0.0.1")
     assert not helpers.is_ip("127.0.0.0.1")
+
+    assert not helpers.is_ip_type("127.0.0.1")
+    assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"))
+    assert not helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=True)
+    assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=False)
+    assert helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"))
+    assert helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=True)
+    assert not helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=False)
+
     assert helpers.is_dns_name("evilcorp.com")
     assert helpers.is_dns_name("evilcorp")
     assert not helpers.is_dns_name("evilcorp", include_local=False)
@@ -212,8 +221,12 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
 
     ipv4_netloc = helpers.make_netloc("192.168.1.1", 80)
     assert ipv4_netloc == "192.168.1.1:80"
-    ipv6_netloc = helpers.make_netloc("dead::beef", "443")
-    assert ipv6_netloc == "[dead::beef]:443"
+    assert helpers.make_netloc("192.168.1.1") == "192.168.1.1"
+    assert helpers.make_netloc(ipaddress.ip_address("192.168.1.1"), None) == "192.168.1.1"
+    assert helpers.make_netloc("dead::beef", "443") == "[dead::beef]:443"
+    assert helpers.make_netloc(ipaddress.ip_address("dead::beef"), 443) == "[dead::beef]:443"
+    assert helpers.make_netloc("dead::beef", None) == "[dead::beef]"
+    assert helpers.make_netloc(ipaddress.ip_address("dead::beef"), None) == "[dead::beef]"
 
     assert helpers.get_file_extension("https://evilcorp.com/evilcorp.com/test/asdf.TXT") == "txt"
     assert helpers.get_file_extension("/etc/conf/test.tar.gz") == "gz"

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/test/test_step_1/test_manager_scope_accuracy.py

@@ -575,7 +575,7 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
         },
     )
 
-    assert len(events) == 10
+    assert len(events) == 12
     assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
     assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"])
     assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
@@ -589,9 +589,9 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
     assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/"])
     assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33" and e.internal == False and e.scope_distance == 0])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888"])
-    assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888"])
-    assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"])
     assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
     assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889"])
     assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
@@ -603,7 +603,7 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.44:8888"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888"])
 
-    assert len(all_events) == 29
+    assert len(all_events) == 31
     assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110" and e.internal == True and e.scope_distance == 0])
     assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
@@ -618,8 +618,10 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888" and e.internal == True and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == True and e.scope_distance == 0])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888" and e.internal == True and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == True and e.scope_distance == 0])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["url"] == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
@@ -660,7 +662,7 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888" and e.internal == True and e.scope_distance == 1])
 
     for _graph_output_events in (graph_output_events, graph_output_batch_events):
-        assert len(_graph_output_events) == 10
+        assert len(_graph_output_events) == 12
         assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
         assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"])
         assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
@@ -674,9 +676,9 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
         assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/"])
         assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33"])
         assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888"])
-        assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"])
+        assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"])
         assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888"])
-        assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"])
+        assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"])
         assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
         assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889"])
         assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/test/test_step_1/test_modules_basic.py

@@ -380,15 +380,14 @@ async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, mo
     scan.modules["dummy"] = dummy(scan)
     events = [e async for e in scan.async_start()]
 
-    assert len(events) == 10
-    for e in events:
-        log.critical(e)
+    assert len(events) == 11
     assert 2 == len([e for e in events if e.type == "SCAN"])
     assert 4 == len([e for e in events if e.type == "DNS_NAME"])
     # one from target and one from speculate
     assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.evilcorp.com"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdf.evilcorp.com"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "asdf.evilcorp.com:443"])
     assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "evilcorp"])
     assert 1 == len([e for e in events if e.type == "FINDING"])
     assert 1 == len([e for e in events if e.type == "URL_UNVERIFIED"])
@@ -397,6 +396,7 @@ async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, mo
         "SCAN": 1,
         "DNS_NAME": 4,
         "URL": 1,
+        "OPEN_TCP_PORT": 1,
         "ORG_STUB": 1,
         "URL_UNVERIFIED": 1,
         "FINDING": 1,
@@ -431,16 +431,17 @@ async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, mo
     assert python_stats.consumed == {
         "DNS_NAME": 4,
         "FINDING": 1,
+        "OPEN_TCP_PORT": 1,
         "ORG_STUB": 1,
         "SCAN": 1,
         "URL": 1,
         "URL_UNVERIFIED": 1,
     }
-    assert python_stats.consumed_total == 9
+    assert python_stats.consumed_total == 10
 
     speculate_stats = scan.stats.module_stats["speculate"]
-    assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1}
-    assert speculate_stats.produced_total == 3
+    assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1, "OPEN_TCP_PORT": 1}
+    assert speculate_stats.produced_total == 4
     assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 3, "URL_UNVERIFIED": 1, "IP_ADDRESS": 3}
     assert speculate_stats.consumed_total == 8
 

{bbot-2.1.0.5004rc0 → bbot-2.1.0.5021rc0}/bbot/test/test_step_1/test_presets.py

@@ -169,6 +169,11 @@ exclude_flags:
 
 def test_preset_scope():
 
+    # test target merging
+    scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]}))
+    assert set([str(h) for h in scan.preset.target.seeds.hosts]) == {"1.2.3.4", "evilcorp.com"}
+    assert set([e.data for e in scan.target]) == {"1.2.3.4", "evilcorp.com"}
+
     blank_preset = Preset()
     blank_preset = blank_preset.bake()
     assert not blank_preset.target