bbot 2.3.0.5438rc0__tar.gz → 2.3.0.5447rc0__tar.gz

{bbot-2.3.0.5438rc0 → bbot-2.3.0.5447rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.3.0.5438rc0
+Version: 2.3.0.5447rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0
@@ -29,6 +29,7 @@ Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
 Requires-Dist: lxml (>=4.9.2,<6.0.0)
 Requires-Dist: mmh3 (>=4.1,<6.0)
 Requires-Dist: omegaconf (>=2.3.0,<3.0.0)
+Requires-Dist: orjson (>=3.10.12,<4.0.0)
 Requires-Dist: psutil (>=5.9.4,<7.0.0)
 Requires-Dist: puremagic (>=1.28,<2.0)
 Requires-Dist: pycryptodome (>=3.17,<4.0)
@@ -280,8 +281,6 @@ config:
     baddns:
       enable_references: True
 
-
-
 ```
 
 </details>

{bbot-2.3.0.5438rc0 → bbot-2.3.0.5447rc0}/README.md

@@ -226,8 +226,6 @@ config:
     baddns:
       enable_references: True
 
-
-
 ```
 
 </details>

{bbot-2.3.0.5438rc0 → bbot-2.3.0.5447rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.0.5438rc"
+__version__ = "v2.3.0.5447rc"
 
 from .scanner import Scanner, Preset

{bbot-2.3.0.5438rc0 → bbot-2.3.0.5447rc0}/bbot/modules/extractous.py

@@ -112,7 +112,7 @@ def extract_text(file_path):
         result = ""
         buffer = reader.read(4096)
         while len(buffer) > 0:
-            result += buffer.decode("utf-8")
+            result += buffer.decode("utf-8", errors="ignore")
             buffer = reader.read(4096)
 
         return result.strip()

{bbot-2.3.0.5438rc0 → bbot-2.3.0.5447rc0}/bbot/modules/httpx.py

@@ -1,5 +1,5 @@
 import re
-import json
+import orjson
 import tempfile
 import subprocess
 from pathlib import Path
@@ -142,11 +142,11 @@ class httpx(BaseModule):
         proxy = self.scan.http_proxy
         if proxy:
             command += ["-http-proxy", proxy]
-        async for line in self.run_process_live(command, input=list(stdin), stderr=subprocess.DEVNULL):
+        async for line in self.run_process_live(command, text=False, input=list(stdin), stderr=subprocess.DEVNULL):
             try:
-                j = json.loads(line)
-            except json.decoder.JSONDecodeError:
-                self.debug(f"Failed to decode line: {line}")
+                j = await self.helpers.run_in_executor(orjson.loads, line)
+            except orjson.JSONDecodeError:
+                self.warning(f"httpx failed to decode line: {line}")
                 continue
 
             url = j.get("url", "")

{bbot-2.3.0.5438rc0 → bbot-2.3.0.5447rc0}/bbot/modules/internal/cloudcheck.py

@@ -72,9 +72,10 @@ class CloudCheck(BaseInterceptModule):
                 base_kwargs["event_type"] = event_type
                 for sig in sigs:
                     matches = []
-                    if event.type == "HTTP_RESPONSE":
-                        matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
-                    elif event.type.startswith("DNS_NAME"):
+                    # TODO: convert this to an excavate YARA hook
+                    # if event.type == "HTTP_RESPONSE":
+                    #     matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
+                    if event.type.startswith("DNS_NAME"):
                         for host in str_hosts_to_check:
                             match = sig.match(host)
                             if match:

{bbot-2.3.0.5438rc0 → bbot-2.3.0.5447rc0}/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py

@@ -8,7 +8,7 @@ class TestCloudCheck(ModuleTestBase):
     modules_overrides = ["httpx", "excavate", "cloudcheck"]
 
     async def setup_after_prep(self, module_test):
-        module_test.set_expect_requests({"uri": "/"}, {"response_data": "<a href='asdf.s3.amazonaws.com'/>"})
+        module_test.set_expect_requests({"uri": "/"}, {"response_data": "<a href='http://asdf.s3.amazonaws.com'/>"})
 
         scan = Scanner(config={"cloudcheck": True})
         await scan._prep()

{bbot-2.3.0.5438rc0 → bbot-2.3.0.5447rc0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.3.0.5438rc"
+version = "v2.3.0.5447rc"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",
@@ -56,6 +56,7 @@ httpx = "^0.27.0"
 puremagic = "^1.28"
 cloudcheck = "^6.0.0.602"
 radixtarget = "^2.0.0.50"
+orjson = "^3.10.12"
 
 [tool.poetry.group.dev.dependencies]
 poetry-dynamic-versioning = ">=0.21.4,<1.5.0"