bbot 2.5.0.6721rc0__tar.gz → 2.5.0.6734rc0__tar.gz

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.5.0.6721rc0
+Version: 2.5.0.6734rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.5.0.6721rc"
+__version__ = "v2.5.0.6734rc"
 
 from .scanner import Scanner, Preset
 

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/core/helpers/depsinstaller/installer.py

@@ -196,7 +196,7 @@ class DepsInstaller:
         deps_common = preloaded["deps"]["common"]
         if deps_common:
             for dep_common in deps_common:
-                if self.setup_status.get(dep_common, False) is True:
+                if self.setup_status.get(dep_common, False) is True and self.deps_behavior != "force_install":
                     log.debug(
                         f'Skipping installation of dependency "{dep_common}" for module "{module}" since it is already installed'
                     )

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/core/helpers/misc.py

@@ -1309,7 +1309,7 @@ def make_netloc(host, port=None):
     return f"{host}:{port}"
 
 
-def which(*executables):
+def which(*executables, path=None):
     """Finds the full path of the first available executable from a list of executables.
 
     Args:
@@ -1325,7 +1325,7 @@ def which(*executables):
     import shutil
 
     for e in executables:
-        location = shutil.which(e)
+        location = shutil.which(e, path=path)
         if location:
             return location
 

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/core/shared_deps.py

@@ -108,6 +108,24 @@ DEP_CHROMIUM = [
         "when": "ansible_facts['os_family'] == 'Debian'",
         "ignore_errors": True,
     },
+    {
+        "name": "Get latest Chromium version (Darwin x86_64)",
+        "uri": {
+            "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2FLAST_CHANGE?alt=media",
+            "return_content": True,
+        },
+        "register": "chromium_version_darwin_x86_64",
+        "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'",
+    },
+    {
+        "name": "Get latest Chromium version (Darwin arm64)",
+        "uri": {
+            "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2FLAST_CHANGE?alt=media",
+            "return_content": True,
+        },
+        "register": "chromium_version_darwin_arm64",
+        "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'",
+    },
     {
         "name": "Download Chromium (Debian)",
         "unarchive": {
@@ -119,6 +137,26 @@ DEP_CHROMIUM = [
         "when": "ansible_facts['os_family'] == 'Debian'",
         "ignore_errors": True,
     },
+    {
+        "name": "Download Chromium (Darwin x86_64)",
+        "unarchive": {
+            "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F{{ chromium_version_darwin_x86_64.content }}%2Fchrome-mac.zip?alt=media",
+            "remote_src": True,
+            "dest": "#{BBOT_TOOLS}",
+            "creates": "#{BBOT_TOOLS}/chrome-mac",
+        },
+        "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'",
+    },
+    {
+        "name": "Download Chromium (Darwin arm64)",
+        "unarchive": {
+            "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2F{{ chromium_version_darwin_arm64.content }}%2Fchrome-mac.zip?alt=media",
+            "remote_src": True,
+            "dest": "#{BBOT_TOOLS}",
+            "creates": "#{BBOT_TOOLS}/chrome-mac",
+        },
+        "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'",
+    },
     # Because Ubuntu is a special snowflake, we have to bend over backwards to fix the chrome sandbox
     # see https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md
     {

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/modules/base.py

@@ -103,6 +103,8 @@ class BaseModule:
 
     # disable the module after this many failed attempts in a row
     _api_failure_abort_threshold = 3
+    # whether to retry on 429s when first pinging the API at scan start
+    _ping_retry_on_http_429 = False
 
     default_discovery_context = "{module} discovered {event.type}: {event.data}"
 
@@ -381,8 +383,9 @@ class BaseModule:
         """
         if url is None:
             url = getattr(self, "ping_url", "")
+        retry_on_http_429 = getattr(self, "_ping_retry_on_http_429", False)
         if url:
-            r = await self.api_request(url)
+            r = await self.api_request(url, retry_on_http_429=retry_on_http_429)
             if getattr(r, "status_code", 0) != 200:
                 response_text = getattr(r, "text", "no response from server")
                 raise ValueError(response_text)
@@ -1200,6 +1203,7 @@ class BaseModule:
             - cancelling after too many failed attempts
         """
         url = args[0] if args else kwargs.pop("url", "")
+        retry_on_http_429 = kwargs.pop("retry_on_http_429", True)
 
         # loop until we have a successful request
         for _ in range(self.api_retries):
@@ -1225,7 +1229,7 @@ class BaseModule:
                 else:
                     # sleep for a bit if we're being rate limited
                     retry_after = self._get_retry_after(r)
-                    if retry_after or status_code == 429:
+                    if (retry_after or status_code == 429) and retry_on_http_429:
                         sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval
                         if retry_after and retry_after > self._429_max_sleep_interval:
                             self.verbose(

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/modules/c99.py

@@ -19,7 +19,7 @@ class c99(subdomain_enum_apikey):
 
     async def ping(self):
         url = f"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json"
-        response = await self.api_request(url)
+        response = await self.api_request(url, retry_on_http_429=False)
         assert response.json()["success"] is True, getattr(response, "text", "no response from server")
 
     async def request_url(self, query):

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/modules/censys.py

@@ -29,7 +29,7 @@ class censys(subdomain_enum_apikey):
 
     async def ping(self):
         url = f"{self.base_url}/v1/account"
-        resp = await self.api_request(url)
+        resp = await self.api_request(url, retry_on_http_429=False)
         d = resp.json()
         assert isinstance(d, dict), f"Invalid response from {url}: {resp}"
         quota = d.get("quota", {})

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/modules/fullhunt.py

@@ -22,7 +22,7 @@ class fullhunt(subdomain_enum_apikey):
 
     async def ping(self):
         url = f"{self.base_url}/auth/status"
-        j = (await self.api_request(url)).json()
+        j = (await self.api_request(url, retry_on_http_429=False)).json()
         remaining = j["user_credits"]["remaining_credits"]
         assert remaining > 0, "No credits remaining"
 

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/modules/gowitness.py

@@ -2,6 +2,7 @@ import os
 import asyncio
 import aiosqlite
 import multiprocessing
+import platform
 from pathlib import Path
 from contextlib import suppress
 from shutil import copyfile, copymode
@@ -23,6 +24,7 @@ class gowitness(BaseModule):
         "output_path": "",
         "social": False,
         "idle_timeout": 1800,
+        "chrome_path": "",
     }
     options_desc = {
         "version": "Gowitness version",
@@ -33,6 +35,7 @@ class gowitness(BaseModule):
         "output_path": "Where to save screenshots",
         "social": "Whether to screenshot social media webpages",
         "idle_timeout": "Skip the current gowitness batch if it stalls for longer than this many seconds",
+        "chrome_path": "Path to chrome executable",
     }
     deps_common = ["chromium"]
     deps_pip = ["aiosqlite"]
@@ -67,29 +70,54 @@ class gowitness(BaseModule):
             self.base_path = Path(output_path) / "gowitness"
         else:
             self.base_path = self.scan.home / "gowitness"
+
         self.chrome_path = None
-        custom_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome"
-        if custom_chrome_path.is_file():
-            self.chrome_path = custom_chrome_path
+        config_chrome_path = self.config.get("chrome_path")
+        if config_chrome_path:
+            config_chrome_path = Path(config_chrome_path)
+            if not config_chrome_path.is_file():
+                return False, f"Could not find custom Chrome path at {config_chrome_path}"
+            self.chrome_path = config_chrome_path
+        else:
+            if platform.system() == "Darwin":
+                bbot_chrome_path = (
+                    self.helpers.tools_dir / "chrome-mac" / "Chromium.app" / "Contents" / "MacOS" / "Chromium"
+                )
+            else:
+                bbot_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome"
+            if bbot_chrome_path.is_file():
+                self.chrome_path = bbot_chrome_path
+
+        # make sure our chrome path works
+        chrome_test_pass = False
+        if self.chrome_path and self.chrome_path.is_file():
+            chrome_test_proc = await self.run_process([str(self.chrome_path), "--version"])
+            if getattr(chrome_test_proc, "returncode", 1) == 0:
+                self.verbose(f"Found chrome executable at {self.chrome_path}")
+                chrome_test_pass = True
+
+        if not chrome_test_pass:
+            # last resort - try to find a working chrome install
+            for binary in ("Google Chrome", "chrome", "chromium", "chromium-browser"):
+                binary_path = self.helpers.which(binary)
+                if binary_path and Path(binary_path).is_file():
+                    chrome_test_proc = await self.run_process([str(binary_path), "--version"])
+                    if getattr(chrome_test_proc, "returncode", 1) == 0:
+                        self.verbose(f"Found chrome executable at {binary_path}")
+                        chrome_test_pass = True
+                        break
+
+        if not chrome_test_pass:
+            return (
+                False,
+                "Failed to set up Google chrome. Please install manually and set `chrome_path`, or try again with --force-deps.",
+            )
 
         # fix ubuntu-specific sandbox bug
         chrome_devel_sandbox = self.helpers.tools_dir / "chrome-linux" / "chrome_sandbox"
         if chrome_devel_sandbox.is_file():
             os.environ["CHROME_DEVEL_SANDBOX"] = str(chrome_devel_sandbox)
 
-        # make sure we have a working chrome install
-        chrome_test_pass = False
-        for binary in ("chrome", "chromium", "chromium-browser", custom_chrome_path):
-            binary_path = self.helpers.which(binary)
-            if binary_path and Path(binary_path).is_file():
-                chrome_test_proc = await self.run_process([binary_path, "--version"])
-                if getattr(chrome_test_proc, "returncode", 1) == 0:
-                    self.verbose(f"Found chrome executable at {binary_path}")
-                    chrome_test_pass = True
-                    break
-        if not chrome_test_pass:
-            return False, "Failed to set up Google chrome. Please install manually or try again with --force-deps."
-
         self.db_path = self.base_path / "gowitness.sqlite3"
         self.screenshot_path = self.base_path / "screenshots"
         self.command = self.construct_command()

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/modules/passivetotal.py

@@ -21,7 +21,7 @@ class passivetotal(subdomain_enum_apikey):
 
     async def ping(self):
         url = f"{self.base_url}/account/quota"
-        j = (await self.api_request(url)).json()
+        j = (await self.api_request(url, retry_on_http_429=False)).json()
         limit = j["user"]["limits"]["search_api"]
         used = j["user"]["counts"]["search_api"]
         assert used < limit, "No quota remaining"

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/bbot/modules/zoomeye.py

@@ -31,7 +31,7 @@ class zoomeye(subdomain_enum_apikey):
 
     async def ping(self):
         url = f"{self.base_url}/resources-info"
-        r = await self.api_request(url)
+        r = await self.api_request(url, retry_on_http_429=False)
         assert int(r.json()["quota_info"]["remain_total_quota"]) > 0, "No quota remaining"
 
     async def handle_event(self, event):

{bbot-2.5.0.6721rc0 → bbot-2.5.0.6734rc0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.5.0.6721rc"
+version = "v2.5.0.6734rc"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",