bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/modules/templates/sql.py

@@ -0,0 +1,95 @@
+from contextlib import suppress
+from sqlmodel import SQLModel
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+
+from bbot.db.sql.models import Event, Scan, Target
+from bbot.modules.output.base import BaseOutputModule
+
+
+class SQLTemplate(BaseOutputModule):
+    meta = {"description": "SQL output module template"}
+    options = {
+        "database": "bbot",
+        "username": "",
+        "password": "",
+        "host": "127.0.0.1",
+        "port": 0,
+    }
+    options_desc = {
+        "database": "The database to use",
+        "username": "The username to use to connect to the database",
+        "password": "The password to use to connect to the database",
+        "host": "The host to use to connect to the database",
+        "port": "The port to use to connect to the database",
+    }
+
+    protocol = ""
+
+    async def setup(self):
+        self.database = self.config.get("database", "bbot")
+        self.username = self.config.get("username", "")
+        self.password = self.config.get("password", "")
+        self.host = self.config.get("host", "127.0.0.1")
+        self.port = self.config.get("port", 0)
+
+        await self.init_database()
+        return True
+
+    async def handle_event(self, event):
+        event_obj = Event(**event.json()).validated
+
+        async with self.async_session() as session:
+            async with session.begin():
+                # insert event
+                session.add(event_obj)
+
+                # if it's a SCAN event, create/update the scan and target
+                if event_obj.type == "SCAN":
+                    event_data = event_obj.get_data()
+                    if not isinstance(event_data, dict):
+                        raise ValueError(f"Invalid data for SCAN event: {event_data}")
+                    scan = Scan(**event_data).validated
+                    await session.merge(scan)  # Insert or update scan
+
+                    target_data = event_data.get("target", {})
+                    if not isinstance(target_data, dict):
+                        raise ValueError(f"Invalid target for SCAN event: {target_data}")
+                    target = Target(**target_data).validated
+                    await session.merge(target)  # Insert or update target
+
+                await session.commit()
+
+    async def create_database(self):
+        pass
+
+    async def init_database(self):
+        await self.create_database()
+
+        # Now create the engine for the actual database
+        self.engine = create_async_engine(self.connection_string())
+        # Create a session factory bound to the engine
+        self.async_session = sessionmaker(self.engine, expire_on_commit=False, class_=AsyncSession)
+
+        # Use the engine directly to create all tables
+        async with self.engine.begin() as conn:
+            await conn.run_sync(SQLModel.metadata.create_all)
+
+    def connection_string(self, mask_password=False):
+        connection_string = f"{self.protocol}://"
+        if self.username:
+            password = self.password
+            if mask_password:
+                password = "****"
+            connection_string += f"{self.username}:{password}"
+        if self.host:
+            connection_string += f"@{self.host}"
+            if self.port:
+                connection_string += f":{self.port}"
+        if self.database:
+            connection_string += f"/{self.database}"
+        return connection_string
+
+    async def cleanup(self):
+        with suppress(Exception):
+            await self.engine.dispose()

bbot/modules/templates/subdomain_enum.py

@@ -20,8 +20,8 @@ class subdomain_enum(BaseModule):
     # whether to reject wildcard DNS_NAMEs
     reject_wildcards = "strict"
 
-    # set qsize to 10. this helps combat rate limiting by ensuring that a query doesn't execute
-    # until the queue is ready to receive its results
+    # set qsize to 10. this helps combat rate limiting by ensuring the next query doesn't execute
+    # until the result from the previous queue have been consumed by the scan
     # we don't use 1 because it causes delays due to the asyncio.sleep; 10 gives us reasonable buffer room
     _qsize = 10
 
@@ -31,6 +31,11 @@ class subdomain_enum(BaseModule):
     #   "lowest_parent": dedupe by lowest parent (lowest parent of www.api.test.evilcorp.com is api.test.evilcorp.com)
     dedup_strategy = "highest_parent"
 
+    # how many results to request per API call
+    page_size = 100
+    # arguments to pass to api_page_iter
+    api_page_iter_kwargs = {}
+
     @property
     def source_pretty_name(self):
         return f"{self.__class__.__name__} API"
@@ -61,9 +66,30 @@ class subdomain_enum(BaseModule):
                             context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
                         )
 
+    async def handle_event_paginated(self, event):
+        query = self.make_query(event)
+        async for result_batch in self.query_paginated(query):
+            for hostname in set(result_batch):
+                try:
+                    hostname = self.helpers.validators.validate_host(hostname)
+                except ValueError as e:
+                    self.verbose(e)
+                    continue
+                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
+                    await self.emit_event(
+                        hostname,
+                        "DNS_NAME",
+                        event,
+                        abort_if=self.abort_if,
+                        context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
+                    )
+
     async def request_url(self, query):
-        url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
-        return await self.request_with_fail_count(url)
+        url = self.make_url(query)
+        return await self.api_request(url)
+
+    def make_url(self, query):
+        return f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
 
     def make_query(self, event):
         query = event.data
@@ -78,30 +104,26 @@ class subdomain_enum(BaseModule):
             if self.scan.in_scope(p):
                 query = p
                 break
-        try:
-            return ".".join([s for s in query.split(".") if s != "_wildcard"])
-        except Exception:
-            self.critical(query)
-            raise
+        return ".".join([s for s in query.split(".") if s != "_wildcard"])
 
-    def parse_results(self, r, query=None):
+    async def parse_results(self, r, query=None):
         json = r.json()
         if json:
             for hostname in json:
                 yield hostname
 
-    async def query(self, query, parse_fn=None, request_fn=None):
-        if parse_fn is None:
-            parse_fn = self.parse_results
+    async def query(self, query, request_fn=None, parse_fn=None):
         if request_fn is None:
             request_fn = self.request_url
+        if parse_fn is None:
+            parse_fn = self.parse_results
         try:
             response = await request_fn(query)
             if response is None:
                 self.info(f'Query "{query}" failed (no response)')
                 return []
             try:
-                results = list(parse_fn(response, query))
+                results = list(await parse_fn(response, query))
             except Exception as e:
                 if response:
                     self.info(
@@ -117,10 +139,23 @@ class subdomain_enum(BaseModule):
         except Exception as e:
             self.info(f"Error retrieving results for {query}: {e}", trace=True)
 
+    async def query_paginated(self, query):
+        url = self.make_url(query)
+        agen = self.api_page_iter(url, page_size=self.page_size, **self.api_page_iter_kwargs)
+        try:
+            async for response in agen:
+                subdomains = await self.parse_results(response, query)
+                self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
+                if not subdomains:
+                    break
+                yield subdomains
+        finally:
+            agen.aclose()
+
     async def _is_wildcard(self, query):
         rdtypes = ("A", "AAAA", "CNAME")
         if self.helpers.is_dns_name(query):
-            for domain, wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query, rdtypes=rdtypes)).items():
+            for wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query, rdtypes=rdtypes)).values():
                 if any(t in wildcard_rdtypes for t in rdtypes):
                     return True
         return False
@@ -134,7 +169,7 @@ class subdomain_enum(BaseModule):
         if any(t.startswith("cloud-") for t in event.tags):
             is_cloud = True
         # reject if it's a cloud resource and not in our target
-        if is_cloud and event not in self.scan.target:
+        if is_cloud and event not in self.scan.target.whitelist:
             return False, "Event is a cloud resource and not a direct target"
         # optionally reject events with wildcards / errors
         if self.reject_wildcards:

bbot/modules/templates/webhook.py

@@ -9,7 +9,6 @@ class WebhookOutputModule(BaseOutputModule):
     """
 
     accept_dupes = False
-    good_status_code = 204
     message_size_limit = 2000
     content_key = "content"
     vuln_severities = ["UNKNOWN", "LOW", "MEDIUM", "HIGH", "CRITICAL"]
@@ -61,7 +60,7 @@ class WebhookOutputModule(BaseOutputModule):
     async def filter_event(self, event):
         if event.type == "VULNERABILITY":
             severity = event.data.get("severity", "UNKNOWN")
-            if not severity in self.allowed_severities:
+            if severity not in self.allowed_severities:
                 return False, f"{severity} is below min_severity threshold"
         return True
 
@@ -94,5 +93,4 @@ class WebhookOutputModule(BaseOutputModule):
         return msg
 
     def evaluate_response(self, response):
-        status_code = getattr(response, "status_code", 0)
-        return status_code == self.good_status_code
+        return getattr(response, "is_success", False)

bbot/modules/trickest.py

@@ -19,53 +19,24 @@ class Trickest(subdomain_enum_apikey):
     }
 
     base_url = "https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be"
+    ping_url = f"{base_url}/dataset"
     dataset_id = "a0a49ca9-03bb-45e0-aa9a-ad59082ebdfc"
     page_size = 50
 
-    async def ping(self):
-        self.headers = {"Authorization": f"Token {self.api_key}"}
-        url = f"{self.base_url}/dataset"
-        response = await self.helpers.request(url, headers=self.headers)
-        status_code = getattr(response, "status_code", 0)
-        if status_code != 200:
-            response_text = getattr(response, "text", "no response from server")
-            return False, response_text
-        return True
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["Authorization"] = f"Token {self.api_key}"
+        return url, kwargs
 
     async def handle_event(self, event):
-        query = self.make_query(event)
-        async for result_batch in self.query(query):
-            for hostname in set(result_batch):
-                try:
-                    hostname = self.helpers.validators.validate_host(hostname)
-                except ValueError as e:
-                    self.verbose(e)
-                    continue
-                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
-                    await self.emit_event(
-                        hostname,
-                        "DNS_NAME",
-                        event,
-                        abort_if=self.abort_if,
-                        context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
-                    )
+        await self.handle_event_paginated(event)
 
-    async def query(self, query):
+    def make_url(self, query):
         url = f"{self.base_url}/view?q=hostname%20~%20%22.{self.helpers.quote(query)}%22"
         url += f"&dataset_id={self.dataset_id}"
         url += "&limit={page_size}&offset={offset}&select=hostname&orderby=hostname"
-        agen = self.helpers.api_page_iter(url, headers=self.headers, page_size=self.page_size)
-        try:
-            async for response in agen:
-                subdomains = self.parse_results(response)
-                self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
-                if not subdomains:
-                    break
-                yield subdomains
-        finally:
-            agen.aclose()
+        return url
 
-    def parse_results(self, j):
+    async def parse_results(self, j, query):
         results = j.get("results", [])
         subdomains = set()
         for item in results:

bbot/modules/trufflehog.py

@@ -1,5 +1,4 @@
 import json
-from pathlib import Path
 from bbot.modules.base import BaseModule
 
 
@@ -14,7 +13,7 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.81.10",
+        "version": "3.84.1",
         "config": "",
         "only_verified": True,
         "concurrency": 8,
@@ -31,7 +30,7 @@ class trufflehog(BaseModule):
         {
             "name": "Download trufflehog",
             "unarchive": {
-                "src": "https://github.com/trufflesecurity/trufflehog/releases/download/v#{BBOT_MODULES_TRUFFLEHOG_VERSION}/trufflehog_#{BBOT_MODULES_TRUFFLEHOG_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz",
+                "src": "https://github.com/trufflesecurity/trufflehog/releases/download/v#{BBOT_MODULES_TRUFFLEHOG_VERSION}/trufflehog_#{BBOT_MODULES_TRUFFLEHOG_VERSION}_#{BBOT_OS_PLATFORM}_#{BBOT_CPU_ARCH}.tar.gz",
                 "include": "trufflehog",
                 "dest": "#{BBOT_TOOLS}",
                 "remote_src": True,
@@ -52,7 +51,7 @@ class trufflehog(BaseModule):
         self.github_token = ""
         if self.deleted_forks:
             self.warning(
-                f"Deleted forks is enabled. Scanning for deleted forks is slooooooowwwww. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours."
+                "Deleted forks is enabled. Scanning for deleted forks is slooooooowwwww. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours."
             )
             for module_name in ("github", "github_codesearch", "github_org", "git_clone"):
                 module_config = self.scan.config.get("modules", {}).get(module_name, {})
@@ -65,7 +64,6 @@ class trufflehog(BaseModule):
             if not self.github_token:
                 self.deleted_forks = False
                 return None, "A github api_key must be provided to the github modules for deleted forks to be scanned"
-        self.processed = set()
         return True
 
     async def filter_event(self, event):
@@ -78,12 +76,8 @@ class trufflehog(BaseModule):
             else:
                 return False, "Deleted forks is not enabled"
         else:
-            path = event.data["path"]
-            for processed in self.processed:
-                processed_path = Path(processed)
-                new_path = Path(path)
-                if new_path.is_relative_to(processed_path):
-                    return False, "Parent folder has already been processed"
+            if "parsed-folder" in event.tags:
+                return False, "Not accepting parsed-folder events"
         return True
 
     async def handle_event(self, event):
@@ -94,11 +88,12 @@ class trufflehog(BaseModule):
                 module = "github-experimental"
         else:
             path = event.data["path"]
-            self.processed.add(path)
             if "git" in event.tags:
                 module = "git"
             elif "docker" in event.tags:
                 module = "docker"
+            elif "postman" in event.tags:
+                module = "postman"
             else:
                 module = "filesystem"
         if event.type == "CODE_REPOSITORY":
@@ -164,6 +159,9 @@ class trufflehog(BaseModule):
         elif module == "docker":
             command.append("docker")
             command.append("--image=file://" + path)
+        elif module == "postman":
+            command.append("postman")
+            command.append("--workspace-paths=" + path)
         elif module == "filesystem":
             command.append("filesystem")
             command.append(path)

bbot/modules/url_manipulation.py

@@ -69,11 +69,11 @@ class url_manipulation(BaseModule):
 
             if subject_response:
                 subject_content = "".join([str(x) for x in subject_response.headers])
-                if subject_response.text != None:
+                if subject_response.text is not None:
                     subject_content += subject_response.text
 
                 if self.rand_string not in subject_content:
-                    if match == False:
+                    if match is False:
                         if str(subject_response.status_code).startswith("2"):
                             if "body" in reasons:
                                 reported_signature = f"Modified URL: {sig[1]}"
@@ -98,7 +98,7 @@ class url_manipulation(BaseModule):
         return False
 
     def format_signature(self, sig, event):
-        if sig[2] == True:
+        if sig[2] is True:
             cleaned_path = event.parsed_url.path.strip("/")
         else:
             cleaned_path = event.parsed_url.path.lstrip("/")

bbot/modules/urlscan.py

@@ -78,5 +78,5 @@ class urlscan(subdomain_enum):
             else:
                 self.debug(f'No results for "{query}"')
         except Exception:
-            self.verbose(f"Error retrieving urlscan results")
+            self.verbose("Error retrieving urlscan results")
         return results

bbot/modules/viewdns.py

@@ -48,7 +48,7 @@ class viewdns(BaseModule):
 
         html = self.helpers.beautifulsoup(content, "html.parser")
         if html is False:
-            self.debug(f"BeautifulSoup returned False")
+            self.debug("BeautifulSoup returned False")
             return results
         found = set()
         for table_row in html.findAll("tr"):

bbot/modules/virustotal.py

@@ -15,37 +15,15 @@ class virustotal(subdomain_enum_apikey):
     options_desc = {"api_key": "VirusTotal API Key"}
 
     base_url = "https://www.virustotal.com/api/v3"
+    api_page_iter_kwargs = {"json": False, "next_key": lambda r: r.json().get("links", {}).get("next", "")}
 
-    async def setup(self):
-        self.api_key = self.config.get("api_key", "")
-        self.headers = {"x-apikey": self.api_key}
-        return await super().setup()
+    def make_url(self, query):
+        return f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains"
 
-    async def ping(self):
-        # virustotal does not have a ping function
-        return
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["x-apikey"] = self.api_key
+        return url, kwargs
 
-    def parse_results(self, r, query):
-        results = set()
+    async def parse_results(self, r, query):
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
-
-    async def query(self, query):
-        results = set()
-        url = f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains"
-        agen = self.helpers.api_page_iter(
-            url, json=False, headers=self.headers, next_key=lambda r: r.json().get("links", {}).get("next", "")
-        )
-        try:
-            async for response in agen:
-                r = self.parse_results(response, query)
-                if not r:
-                    break
-                results.update(r)
-        finally:
-            agen.aclose()
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)

bbot/modules/wafw00f.py

@@ -52,7 +52,7 @@ class wafw00f(BaseModule):
                     context=f"{{module}} scanned {url} and identified {{event.type}}: {waf}",
                 )
         else:
-            if self.config.get("generic_detect") == True:
+            if self.config.get("generic_detect") is True:
                 generic = await self.helpers.run_in_executor(WW.genericdetect)
                 if generic:
                     waf = "generic detection"

bbot/modules/wayback.py

@@ -10,7 +10,7 @@ class wayback(subdomain_enum):
     meta = {
         "description": "Query archive.org's API for subdomains",
         "created_date": "2022-04-01",
-        "author": "@pmueller",
+        "author": "@liquidsec",
     }
     options = {"urls": False, "garbage_threshold": 10}
     options_desc = {

bbot/modules/wpscan.py

@@ -14,26 +14,26 @@ class wpscan(BaseModule):
 
     options = {
         "api_key": "",
-        "enumerate": "vp,vt,tt,cb,dbe,u,m",
+        "enumerate": "vp,vt,cb,dbe",
         "threads": 5,
-        "request_timeout": 60,
-        "connection_timeout": 30,
+        "request_timeout": 5,
+        "connection_timeout": 2,
         "disable_tls_checks": True,
         "force": False,
     }
     options_desc = {
         "api_key": "WPScan API Key",
-        "enumerate": "Enumeration Process see wpscan help documentation (default: vp,vt,tt,cb,dbe,u,m)",
+        "enumerate": "Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)",
         "threads": "How many wpscan threads to spawn (default is 5)",
-        "request_timeout": "The request timeout in seconds (default 60)",
-        "connection_timeout": "The connection timeout in seconds (default 30)",
+        "request_timeout": "The request timeout in seconds (default 5)",
+        "connection_timeout": "The connection timeout in seconds (default 2)",
         "disable_tls_checks": "Disables the SSL/TLS certificate verification (Default True)",
         "force": "Do not check if the target is running WordPress or returns a 403",
     }
     deps_apt = ["curl", "make", "gcc"]
     deps_ansible = [
         {
-            "name": "Install Ruby Deps (Debian/Ubuntu)",
+            "name": "Install Ruby Deps (Debian)",
             "package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"},
             "become": True,
             "when": "ansible_facts['os_family'] == 'Debian'",
@@ -48,7 +48,13 @@ class wpscan(BaseModule):
             "name": "Install Ruby Deps (Fedora)",
             "package": {"name": ["rubygems", "ruby-devel"], "state": "present"},
             "become": True,
-            "when": "ansible_facts['os_family'] == 'Fedora'",
+            "when": "ansible_facts['os_family'] == 'RedHat'",
+        },
+        {
+            "name": "Install Ruby Deps (Alpine)",
+            "package": {"name": ["ruby-dev", "ruby-bundler"], "state": "present"},
+            "become": True,
+            "when": "ansible_facts['os_family'] == 'Alpine'",
         },
         {
             "name": "Install wpscan gem",
@@ -61,11 +67,11 @@ class wpscan(BaseModule):
         self.processed = set()
         self.ignore_events = ["xmlrpc", "readme"]
         self.api_key = self.config.get("api_key", "")
-        self.enumerate = self.config.get("enumerate", "vp,vt,tt,cb,dbe,u,m")
+        self.enumerate = self.config.get("enumerate", "vp,vt,cb,dbe")
         self.proxy = self.scan.web_config.get("http_proxy", "")
         self.threads = self.config.get("threads", 5)
-        self.request_timeout = self.config.get("request_timeout", 60)
-        self.connection_timeout = self.config.get("connection_timeout", 30)
+        self.request_timeout = self.config.get("request_timeout", 5)
+        self.connection_timeout = self.config.get("connection_timeout", 2)
         self.disable_tls_checks = self.config.get("disable_tls_checks", True)
         self.force = self.config.get("force", False)
         return True

bbot/modules/zoomeye.py

@@ -22,13 +22,16 @@ class zoomeye(subdomain_enum_apikey):
 
     async def setup(self):
         self.max_pages = self.config.get("max_pages", 20)
-        self.headers = {"API-KEY": self.config.get("api_key", "")}
         self.include_related = self.config.get("include_related", False)
         return await super().setup()
 
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["API-KEY"] = self.api_key
+        return url, kwargs
+
     async def ping(self):
         url = f"{self.base_url}/resources-info"
-        r = await self.helpers.request(url, headers=self.headers)
+        r = await self.api_request(url)
         assert int(r.json()["quota_info"]["remain_total_quota"]) > 0, "No quota remaining"
 
     async def handle_event(self, event):
@@ -54,10 +57,10 @@ class zoomeye(subdomain_enum_apikey):
         query_type = 0 if self.include_related else 1
         url = f"{self.base_url}/domain/search?q={self.helpers.quote(query)}&type={query_type}&page=" + "{page}"
         i = 0
-        agen = self.helpers.api_page_iter(url, headers=self.headers)
+        agen = self.api_page_iter(url)
         try:
             async for j in agen:
-                r = list(self.parse_results(j))
+                r = list(await self.parse_results(j))
                 if r:
                     results.update(set(r))
                 if not r or i >= (self.max_pages - 1):
@@ -67,6 +70,8 @@ class zoomeye(subdomain_enum_apikey):
             agen.aclose()
         return results
 
-    def parse_results(self, r):
+    async def parse_results(self, r):
+        results = set()
         for entry in r.get("list", []):
-            yield entry["name"]
+            results.add(entry["name"])
+        return results

bbot/presets/baddns-thorough.yml

@@ -0,0 +1,12 @@
+description: Run all baddns modules and submodules.
+
+
+modules:
+  - baddns
+  - baddns_zone
+  - baddns_direct
+
+config:
+  modules:
+    baddns:
+      enabled_submodules: [CNAME,references,MX,NS,TXT]

bbot/presets/fast.yml

@@ -0,0 +1,16 @@
+description: Scan only the provided targets as fast as possible - no extra discovery
+
+exclude_modules:
+  - excavate
+
+config:
+  # only scan the exact targets specified
+  scope:
+    strict: true
+  # speed up dns resolution by doing A/AAAA only - not MX/NS/SRV/etc
+  dns:
+    minimal: true
+  # essential speculation only
+  modules:
+    speculate:
+      essential_only: true

bbot/presets/kitchen-sink.yml

@@ -10,6 +10,7 @@ include:
   - paramminer
   - dirbust-light
   - web-screenshots
+  - baddns-thorough
 
 config:
   modules:

bbot/presets/spider.yml

@@ -3,6 +3,10 @@ description: Recursive web spider
 modules:
   - httpx
 
+blacklist:
+  # Prevent spider from invalidating sessions by logging out
+  - "RE:/.*(sign|log)[_-]?out"
+
 config:
   web:
     # how many links to follow in a row