PyPI - bbot - Versions diffs - 2.6.0.6879rc0__py3-none-any.whl → 2.7.2.7254rc0__py3-none-any.whl - Mend

bbot 2.6.0.6879rc0py3-none-any.whl → 2.7.2.7254rc0py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of bbot might be problematic. Click here for more details.

Files changed (75) hide show

bbot/__init__.py +1 -1
bbot/core/engine.py +1 -1
bbot/core/flags.py +1 -0
bbot/core/helpers/bloom.py +6 -7
bbot/core/helpers/dns/dns.py +0 -1
bbot/core/helpers/dns/engine.py +0 -2
bbot/core/helpers/files.py +2 -2
bbot/core/helpers/git.py +17 -0
bbot/core/helpers/misc.py +1 -0
bbot/core/helpers/ntlm.py +0 -2
bbot/core/helpers/regex.py +1 -1
bbot/core/modules.py +0 -54
bbot/defaults.yml +4 -2
bbot/modules/apkpure.py +1 -1
bbot/modules/base.py +11 -5
bbot/modules/dnsbimi.py +1 -4
bbot/modules/dnsdumpster.py +35 -52
bbot/modules/dnstlsrpt.py +0 -6
bbot/modules/docker_pull.py +1 -1
bbot/modules/emailformat.py +17 -1
bbot/modules/filedownload.py +1 -1
bbot/modules/git_clone.py +47 -22
bbot/modules/gitdumper.py +4 -14
bbot/modules/github_workflows.py +1 -1
bbot/modules/gitlab_com.py +31 -0
bbot/modules/gitlab_onprem.py +84 -0
bbot/modules/gowitness.py +0 -6
bbot/modules/graphql_introspection.py +5 -2
bbot/modules/httpx.py +2 -0
bbot/modules/iis_shortnames.py +0 -7
bbot/modules/internal/unarchive.py +9 -3
bbot/modules/lightfuzz/lightfuzz.py +5 -1
bbot/modules/nuclei.py +1 -1
bbot/modules/output/base.py +0 -5
bbot/modules/postman_download.py +1 -1
bbot/modules/retirejs.py +232 -0
bbot/modules/securitytxt.py +0 -3
bbot/modules/subdomaincenter.py +1 -16
bbot/modules/telerik.py +6 -1
bbot/modules/templates/gitlab.py +98 -0
bbot/modules/trufflehog.py +1 -1
bbot/scanner/manager.py +7 -4
bbot/scanner/scanner.py +1 -1
bbot/scripts/benchmark_report.py +433 -0
bbot/test/benchmarks/__init__.py +2 -0
bbot/test/benchmarks/test_bloom_filter_benchmarks.py +105 -0
bbot/test/benchmarks/test_closest_match_benchmarks.py +76 -0
bbot/test/benchmarks/test_event_validation_benchmarks.py +438 -0
bbot/test/benchmarks/test_excavate_benchmarks.py +291 -0
bbot/test/benchmarks/test_ipaddress_benchmarks.py +143 -0
bbot/test/benchmarks/test_weighted_shuffle_benchmarks.py +70 -0
bbot/test/test_step_1/test_bbot_fastapi.py +2 -2
bbot/test/test_step_1/test_events.py +0 -1
bbot/test/test_step_1/test_scan.py +1 -8
bbot/test/test_step_2/module_tests/base.py +6 -1
bbot/test/test_step_2/module_tests/test_module_dnsbimi.py +2 -1
bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py +3 -5
bbot/test/test_step_2/module_tests/test_module_emailformat.py +1 -1
bbot/test/test_step_2/module_tests/test_module_emails.py +2 -2
bbot/test/test_step_2/module_tests/test_module_excavate.py +35 -6
bbot/test/test_step_2/module_tests/test_module_gitlab_com.py +66 -0
bbot/test/test_step_2/module_tests/{test_module_gitlab.py → test_module_gitlab_onprem.py} +4 -69
bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +2 -2
bbot/test/test_step_2/module_tests/test_module_retirejs.py +159 -0
bbot/test/test_step_2/module_tests/test_module_telerik.py +1 -1
{bbot-2.6.0.6879rc0.dist-info → bbot-2.7.2.7254rc0.dist-info}/METADATA +7 -4
{bbot-2.6.0.6879rc0.dist-info → bbot-2.7.2.7254rc0.dist-info}/RECORD +70 -60
{bbot-2.6.0.6879rc0.dist-info → bbot-2.7.2.7254rc0.dist-info}/WHEEL +1 -1
bbot/modules/censys.py +0 -98
bbot/modules/gitlab.py +0 -141
bbot/modules/zoomeye.py +0 -77
bbot/test/test_step_2/module_tests/test_module_censys.py +0 -83
bbot/test/test_step_2/module_tests/test_module_zoomeye.py +0 -35
{bbot-2.6.0.6879rc0.dist-info → bbot-2.7.2.7254rc0.dist-info}/entry_points.txt +0 -0
{bbot-2.6.0.6879rc0.dist-info → bbot-2.7.2.7254rc0.dist-info/licenses}/LICENSE +0 -0

bbot/test/benchmarks/test_excavate_benchmarks.py ADDED Viewed

@@ -0,0 +1,291 @@
+import pytest
+import asyncio
+from bbot.scanner import Scanner
+class TestExcavateDirectBenchmarks:
+    """
+    Direct benchmark tests for Excavate module operations.
+    These tests measure the performance of excavate's core YARA processing
+    by calling the excavate.search() method directly with specific text sizes
+    in both single-threaded and parallel asyncio tasks to test the GIL sidestep feature of YARA.
+    """
+    # Number of text segments per test
+    TEXT_SEGMENTS_COUNT = 100
+    # Prescribed sizes for deterministic benchmarking (in bytes)
+    SMALL_SIZE = 4096  # 4KB
+    LARGE_SIZE = 5242880  # 5MB
+    def _generate_text_segments(self, target_size, count):
+        """Generate a list of text segments of the specified size"""
+        segments = []
+        for i in range(count):
+            # Generate realistic content that excavate can work with
+            base_content = self._generate_realistic_content(i)
+            # Pad to the exact target size with deterministic content
+            remaining_size = target_size - len(base_content)
+            if remaining_size > 0:
+                # Use deterministic padding pattern
+                padding_pattern = "Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. "
+                padding_repeats = (remaining_size // len(padding_pattern)) + 1
+                padding = (padding_pattern * padding_repeats)[:remaining_size]
+                content = base_content + padding
+            else:
+                content = base_content[:target_size]
+            segments.append(content)
+        return segments
+    def _generate_realistic_content(self, index):
+        """Generate realistic content that excavate can extract from"""
+        return f"""
+        <html>
+        <head>
+            <title>Test Content {index}</title>
+            <script src="https://api{index}.example.com/js/app.js"></script>
+        </head>
+        <body>
+            <h1>Page {index}</h1>
+            <!-- URLs and subdomains -->
+            <a href="https://www{index}.example.com/page{index}">Link {index}</a>
+            <a href="https://cdn{index}.example.com/assets/">CDN {index}</a>
+            <img src="https://img{index}.example.com/photo{index}.jpg" />
+            <!-- Forms with parameters -->
+            <form action="/search{index}" method="GET">
+                <input type="text" name="query{index}" value="test{index}">
+                <input type="hidden" name="token{index}" value="abc123{index}">
+                <button type="submit">Search</button>
+            </form>
+            <!-- API endpoints -->
+            <script>
+                fetch('https://api{index}.example.com/v1/users/{index}')
+                    .then(response => response.json())
+                    .then(data => console.log(data));
+                // WebSocket connection
+                const ws = new WebSocket('wss://realtime{index}.example.com/socket');
+            </script>
+            <!-- Various protocols -->
+            <p>FTP: ftp://ftp{index}.example.com:21/files/</p>
+            <p>SSH: ssh://server{index}.example.com:22/</p>
+            <p>Email: contact{index}@example.com</p>
+            <!-- JSON data -->
+            <script type="application/json">
+            {{
+                "apiEndpoint{index}": "https://api{index}.example.com/data",
+                "parameter{index}": "value{index}",
+                "secretKey{index}": "sk_test_{index}_abcdef123456"
+            }}
+            </script>
+            <!-- Comments with URLs -->
+            <!-- https://hidden{index}.example.com/admin -->
+            <!-- TODO: Check https://internal{index}.example.com/debug -->
+        </body>
+        </html>
+        """
+    async def _run_excavate_single_thread(self, text_segments):
+        """Run excavate processing in single thread"""
+        # Create scanner and initialize excavate
+        scan = Scanner("example.com", modules=["httpx"], config={"excavate": True})
+        await scan._prep()
+        excavate_module = scan.modules.get("excavate")
+        if not excavate_module:
+            raise RuntimeError("Excavate module not found")
+        # Track events emitted by excavate
+        emitted_events = []
+        async def track_emit_event(event_data, *args, **kwargs):
+            emitted_events.append(event_data)
+        excavate_module.emit_event = track_emit_event
+        # Process all text segments sequentially
+        results = []
+        for i, text_segment in enumerate(text_segments):
+            # Create a mock HTTP_RESPONSE event
+            mock_event = scan.make_event(
+                {
+                    "url": f"https://example.com/test/{i}",
+                    "method": "GET",
+                    "body": text_segment,
+                    "header-dict": {"Content-Type": ["text/html"]},
+                    "raw_header": "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
+                    "status_code": 200,
+                },
+                "HTTP_RESPONSE",
+                parent=scan.root_event,
+            )
+            # Process with excavate
+            await excavate_module.search(text_segment, mock_event, "text/html", f"Single thread benchmark {i}")
+            results.append(f"processed_{i}")
+        return results, emitted_events
+    async def _run_excavate_parallel_tasks(self, text_segments):
+        """Run excavate processing with parallel asyncio tasks"""
+        # Create scanner and initialize excavate
+        scan = Scanner("example.com", modules=["httpx"], config={"excavate": True})
+        await scan._prep()
+        excavate_module = scan.modules.get("excavate")
+        if not excavate_module:
+            raise RuntimeError("Excavate module not found")
+        # Define async task to process a single text segment
+        async def process_segment(segment_index, text_segment):
+            mock_event = scan.make_event(
+                {
+                    "url": f"https://example.com/parallel/{segment_index}",
+                    "method": "GET",
+                    "body": text_segment,
+                    "header-dict": {"Content-Type": ["text/html"]},
+                    "raw_header": "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
+                    "status_code": 200,
+                },
+                "HTTP_RESPONSE",
+                parent=scan.root_event,
+            )
+            await excavate_module.search(
+                text_segment, mock_event, "text/html", f"Parallel benchmark task {segment_index}"
+            )
+            return f"processed_{segment_index}"
+        # Create all tasks and run them concurrently
+        tasks = [process_segment(i, text_segment) for i, text_segment in enumerate(text_segments)]
+        # Run all tasks in parallel
+        results = await asyncio.gather(*tasks)
+        return results
+    # Single Thread Tests
+    @pytest.mark.benchmark(group="excavate_single_small")
+    def test_excavate_single_thread_small(self, benchmark):
+        """Benchmark excavate single thread processing with small (4KB) segments"""
+        text_segments = self._generate_text_segments(self.SMALL_SIZE, self.TEXT_SEGMENTS_COUNT)
+        def run_test():
+            return asyncio.run(self._run_excavate_single_thread(text_segments))
+        result, events = benchmark(run_test)
+        assert len(result) == self.TEXT_SEGMENTS_COUNT
+        total_size_mb = (self.SMALL_SIZE * self.TEXT_SEGMENTS_COUNT) / (1024 * 1024)
+        # Count events by type
+        total_events = len(events)
+        url_events = len([e for e in events if e.type == "URL_UNVERIFIED"])
+        dns_events = len([e for e in events if e.type == "DNS_NAME"])
+        email_events = len([e for e in events if e.type == "EMAIL_ADDRESS"])
+        protocol_events = len([e for e in events if e.type == "PROTOCOL"])
+        finding_events = len([e for e in events if e.type == "FINDING"])
+        print("\n✅ Single-thread small segments benchmark completed")
+        print(f"📊 Processed {len(result):,} segments of {self.SMALL_SIZE / 1024:.0f}KB each")
+        print(f"📊 Total size processed: {total_size_mb:.1f} MB")
+        print(f"📊 Total events: {total_events}")
+        print(f"📊 URL events: {url_events}")
+        print(f"📊 DNS events: {dns_events}")
+        print(f"📊 Email events: {email_events}")
+        print(f"📊 Protocol events: {protocol_events}")
+        print(f"📊 Finding events: {finding_events}")
+        # Validate that excavate actually found and processed content
+        assert total_events > 0, "Expected to find some events from excavate"
+        assert url_events > 0 or dns_events > 0 or protocol_events > 0, (
+            "Expected excavate to find URLs, DNS names, or protocols"
+        )
+    @pytest.mark.benchmark(group="excavate_single_large")
+    def test_excavate_single_thread_large(self, benchmark):
+        """Benchmark excavate single thread processing with large (10MB) segments"""
+        text_segments = self._generate_text_segments(self.LARGE_SIZE, self.TEXT_SEGMENTS_COUNT)
+        def run_test():
+            return asyncio.run(self._run_excavate_single_thread(text_segments))
+        result, events = benchmark(run_test)
+        assert len(result) == self.TEXT_SEGMENTS_COUNT
+        total_size_mb = (self.LARGE_SIZE * self.TEXT_SEGMENTS_COUNT) / (1024 * 1024)
+        # Count events by type
+        total_events = len(events)
+        url_events = len([e for e in events if e.type == "URL_UNVERIFIED"])
+        dns_events = len([e for e in events if e.type == "DNS_NAME"])
+        email_events = len([e for e in events if e.type == "EMAIL_ADDRESS"])
+        protocol_events = len([e for e in events if e.type == "PROTOCOL"])
+        finding_events = len([e for e in events if e.type == "FINDING"])
+        print("\n✅ Single-thread large segments benchmark completed")
+        print(f"📊 Processed {len(result):,} segments of {self.LARGE_SIZE / (1024 * 1024):.0f}MB each")
+        print(f"📊 Total size processed: {total_size_mb:.1f} MB")
+        print(f"📊 Total events: {total_events}")
+        print(f"📊 URL events: {url_events}")
+        print(f"📊 DNS events: {dns_events}")
+        print(f"📊 Email events: {email_events}")
+        print(f"📊 Protocol events: {protocol_events}")
+        print(f"📊 Finding events: {finding_events}")
+        # Validate that excavate actually found and processed content
+        assert total_events > 0, "Expected to find some events from excavate"
+        assert url_events > 0 or dns_events > 0 or protocol_events > 0, (
+            "Expected excavate to find URLs, DNS names, or protocols"
+        )
+    # Parallel Tests
+    @pytest.mark.benchmark(group="excavate_parallel_small")
+    def test_excavate_parallel_tasks_small(self, benchmark):
+        """Benchmark excavate parallel processing with small (4KB) segments"""
+        text_segments = self._generate_text_segments(self.SMALL_SIZE, self.TEXT_SEGMENTS_COUNT)
+        def run_test():
+            return asyncio.run(self._run_excavate_parallel_tasks(text_segments))
+        result = benchmark(run_test)
+        assert len(result) == self.TEXT_SEGMENTS_COUNT
+        total_size_mb = (self.SMALL_SIZE * self.TEXT_SEGMENTS_COUNT) / (1024 * 1024)
+        print("\n✅ Parallel small segments benchmark completed")
+        print(f"📊 Processed {len(result):,} segments of {self.SMALL_SIZE / 1024:.0f}KB each in parallel")
+        print(f"📊 Total size processed: {total_size_mb:.1f} MB")
+        print("📊 Tasks executed concurrently to test YARA GIL sidestep")
+        # Basic assertion that excavate is actually working (should find URLs in our test content)
+        assert len(result) > 0, "Expected excavate to process all segments"
+    @pytest.mark.benchmark(group="excavate_parallel_large")
+    def test_excavate_parallel_tasks_large(self, benchmark):
+        """Benchmark excavate parallel processing with large (10MB) segments to test YARA GIL sidestep"""
+        text_segments = self._generate_text_segments(self.LARGE_SIZE, self.TEXT_SEGMENTS_COUNT)
+        def run_test():
+            return asyncio.run(self._run_excavate_parallel_tasks(text_segments))
+        result = benchmark(run_test)
+        assert len(result) == self.TEXT_SEGMENTS_COUNT
+        total_size_mb = (self.LARGE_SIZE * self.TEXT_SEGMENTS_COUNT) / (1024 * 1024)
+        print("\n✅ Parallel large segments benchmark completed")
+        print(f"📊 Processed {len(result):,} segments of {self.LARGE_SIZE / (1024 * 1024):.0f}MB each in parallel")
+        print(f"📊 Total size processed: {total_size_mb:.1f} MB")
+        print("📊 Tasks executed concurrently to test YARA GIL sidestep")
+        # Basic assertion that excavate is actually working (should find URLs in our test content)
+        assert len(result) > 0, "Expected excavate to process all segments"

bbot/test/benchmarks/test_ipaddress_benchmarks.py ADDED Viewed

@@ -0,0 +1,143 @@
+import pytest
+import random
+import string
+from bbot.core.helpers.misc import make_ip_type, is_ip
+class TestIPAddressBenchmarks:
+    """
+    Benchmark tests for IP address processing operations.
+    These tests measure the performance of BBOT-level IP functions which are
+    critical for network scanning efficiency and could benefit from different
+    underlying implementations.
+    """
+    def setup_method(self):
+        """Setup common test data"""
+        # Set deterministic seed for consistent benchmark results
+        random.seed(42)  # Fixed seed for reproducible results
+        # Generate test data of different types and sizes
+        self.valid_ips = self._generate_valid_ips()
+        self.invalid_ips = self._generate_invalid_ips()
+        self.mixed_data = self._generate_mixed_data()
+    def _generate_valid_ips(self):
+        """Generate valid IP addresses for testing"""
+        valid_ips = []
+        # IPv4 addresses
+        for i in range(1000):
+            valid_ips.append(
+                f"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(1, 254)}"
+            )
+        # IPv6 addresses
+        for i in range(500):
+            ipv6_parts = []
+            for j in range(8):
+                ipv6_parts.append(f"{random.randint(0, 65535):x}")
+            valid_ips.append(":".join(ipv6_parts))
+        # Network addresses
+        for i in range(500):
+            base_ip = f"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.0"
+            valid_ips.append(f"{base_ip}/{random.randint(8, 30)}")
+        # IP ranges
+        for i in range(200):
+            start_ip = (
+                f"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(1, 200)}"
+            )
+            end_ip = f"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(201, 254)}"
+            valid_ips.append(f"{start_ip}-{end_ip}")
+        return valid_ips
+    def _generate_invalid_ips(self):
+        """Generate invalid IP addresses for testing"""
+        invalid_ips = []
+        # Malformed IPv4
+        for i in range(500):
+            invalid_ips.append(
+                f"{random.randint(256, 999)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}"
+            )
+            invalid_ips.append(f"{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}")
+            invalid_ips.append(
+                f"{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}"
+            )
+        # Malformed IPv6
+        for i in range(300):
+            ipv6_parts = []
+            for j in range(random.randint(5, 10)):  # Wrong number of parts
+                ipv6_parts.append(f"{random.randint(0, 65535):x}")
+            invalid_ips.append(":".join(ipv6_parts))
+        # Random strings
+        for i in range(200):
+            length = random.randint(5, 20)
+            invalid_ips.append("".join(random.choices(string.ascii_letters + string.digits, k=length)))
+        return invalid_ips
+    def _generate_mixed_data(self):
+        """Generate mixed valid/invalid data for realistic testing"""
+        mixed = []
+        mixed.extend(self.valid_ips[:500])  # First 500 valid
+        mixed.extend(self.invalid_ips[:500])  # First 500 invalid
+        # Use deterministic shuffle with fixed seed for consistent results
+        random.seed(42)  # Reset seed before shuffle
+        random.shuffle(mixed)  # Shuffle for realistic distribution
+        return mixed
+    @pytest.mark.benchmark(group="ip_validation")
+    def test_is_ip_performance(self, benchmark):
+        """Benchmark IP validation performance with mixed data"""
+        def validate_ips():
+            valid_count = 0
+            for ip in self.mixed_data:
+                if is_ip(ip):
+                    valid_count += 1
+            return valid_count
+        result = benchmark(validate_ips)
+        assert result > 0
+    @pytest.mark.benchmark(group="ip_type_detection")
+    def test_make_ip_type_performance(self, benchmark):
+        """Benchmark IP type detection performance"""
+        def detect_ip_types():
+            type_count = 0
+            for ip in self.valid_ips:
+                try:
+                    make_ip_type(ip)
+                    type_count += 1
+                except Exception:
+                    pass
+            return type_count
+        result = benchmark(detect_ip_types)
+        assert result > 0
+    @pytest.mark.benchmark(group="ip_processing")
+    def test_mixed_ip_operations(self, benchmark):
+        """Benchmark combined IP validation + type detection"""
+        def process_ips():
+            processed = 0
+            for ip in self.mixed_data:
+                if is_ip(ip):
+                    try:
+                        make_ip_type(ip)
+                        processed += 1
+                    except Exception:
+                        pass
+            return processed
+        result = benchmark(process_ips)
+        assert result > 0

bbot/test/benchmarks/test_weighted_shuffle_benchmarks.py ADDED Viewed

@@ -0,0 +1,70 @@
+import pytest
+import random
+from bbot.core.helpers.misc import weighted_shuffle
+class TestWeightedShuffleBenchmarks:
+    """
+    Benchmark tests for weighted_shuffle operations.
+    This function is critical for BBOT's queue management, where it shuffles
+    incoming queues based on module priority weights. Performance here directly
+    impacts scan throughput and responsiveness.
+    """
+    def setup_method(self):
+        """Setup common test data"""
+        # Set deterministic seed for consistent benchmark results
+        random.seed(42)  # Fixed seed for reproducible results
+        # Generate test data of different sizes and complexity
+        self.small_data = self._generate_small_dataset()
+        self.medium_data = self._generate_medium_dataset()
+        self.large_data = self._generate_large_dataset()
+        self.priority_weights = self._generate_priority_weights()
+    def _generate_small_dataset(self):
+        """Generate small dataset (like few modules)"""
+        return {"items": ["module_a", "module_b", "module_c"], "weights": [0.6, 0.3, 0.1]}
+    def _generate_medium_dataset(self):
+        """Generate medium dataset (like typical scan)"""
+        items = [f"module_{i}" for i in range(20)]
+        weights = [random.uniform(0.1, 1.0) for _ in range(20)]
+        return {"items": items, "weights": weights}
+    def _generate_large_dataset(self):
+        """Generate large dataset (like complex scan with many modules)"""
+        items = [f"module_{i}" for i in range(100)]
+        weights = [random.uniform(0.1, 1.0) for _ in range(100)]
+        return {"items": items, "weights": weights}
+    def _generate_priority_weights(self):
+        """Generate realistic priority weights (like BBOT module priorities)"""
+        # BBOT uses priorities 1-5, where lower priority = higher weight
+        # Weights are calculated as [5] + [6 - m.priority for m in modules]
+        priorities = [5] + [6 - p for p in [1, 2, 3, 4, 5]] * 20  # 5 + 5*20 = 105 items
+        items = [f"queue_{i}" for i in range(len(priorities))]
+        return {"items": items, "weights": priorities}
+    @pytest.mark.benchmark(group="weighted_shuffle")
+    def test_typical_queue_shuffle(self, benchmark):
+        """Benchmark weighted shuffle with typical BBOT scan workload"""
+        def shuffle_typical():
+            return weighted_shuffle(self.medium_data["items"], self.medium_data["weights"])
+        result = benchmark(shuffle_typical)
+        assert len(result) == 20
+        assert all(item in result for item in self.medium_data["items"])
+    @pytest.mark.benchmark(group="weighted_shuffle")
+    def test_priority_queue_shuffle(self, benchmark):
+        """Benchmark weighted shuffle with realistic BBOT priority weights"""
+        def shuffle_priorities():
+            return weighted_shuffle(self.priority_weights["items"], self.priority_weights["weights"])
+        result = benchmark(shuffle_priorities)
+        assert len(result) == len(self.priority_weights["items"])
+        assert all(item in result for item in self.priority_weights["items"])

bbot/test/test_step_1/test_bbot_fastapi.py CHANGED Viewed

@@ -22,8 +22,8 @@ def test_bbot_multiprocess(bbot_httpserver):
     queue = multiprocessing.Queue()
     events_process = multiprocessing.Process(target=run_bbot_multiprocess, args=(queue,))
     events_process.start()
-    events_process.join()
-    events = queue.get()
+    events_process.join(timeout=300)
+    events = queue.get(timeout=10)
     assert len(events) >= 3
     scan_events = [e for e in events if e["type"] == "SCAN"]
     assert len(scan_events) == 2

bbot/test/test_step_1/test_events.py CHANGED Viewed

@@ -209,7 +209,6 @@ async def test_events(events, helpers):
     javascript_event = scan.make_event("http://evilcorp.com/asdf/a.js?b=c#d", "URL_UNVERIFIED", parent=scan.root_event)
     assert "extension-js" in javascript_event.tags
     await scan.ingress_module.handle_event(javascript_event)
-    assert "httpx-only" in javascript_event.tags
     # scope distance
     event1 = scan.make_event("1.2.3.4", dummy=True)

bbot/test/test_step_1/test_scan.py CHANGED Viewed

@@ -111,7 +111,6 @@ async def test_task_scan_handle_event_timeout(bbot_scanner):
     class LongBatchModule(BaseModule):
         watched_events = ["IP_ADDRESS"]
         handled_event = False
-        canceled = False
         _name = "long_batch"
         _batch_size = 2
@@ -147,24 +146,18 @@ async def test_task_scan_handle_event_timeout(bbot_scanner):
 @pytest.mark.asyncio
 async def test_url_extension_handling(bbot_scanner):
-    scan = bbot_scanner(config={"url_extension_blacklist": ["css"], "url_extension_httpx_only": ["js"]})
+    scan = bbot_scanner(config={"url_extension_blacklist": ["css"]})
     await scan._prep()
     assert scan.url_extension_blacklist == {"css"}
-    assert scan.url_extension_httpx_only == {"js"}
     good_event = scan.make_event("https://evilcorp.com/a.txt", "URL", tags=["status-200"], parent=scan.root_event)
     bad_event = scan.make_event("https://evilcorp.com/a.css", "URL", tags=["status-200"], parent=scan.root_event)
-    httpx_event = scan.make_event("https://evilcorp.com/a.js", "URL", tags=["status-200"], parent=scan.root_event)
     assert "blacklisted" not in bad_event.tags
-    assert "httpx-only" not in httpx_event.tags
     result = await scan.ingress_module.handle_event(good_event)
     assert result is None
     result, reason = await scan.ingress_module.handle_event(bad_event)
     assert result is False
     assert reason == "event is blacklisted"
     assert "blacklisted" in bad_event.tags
-    result = await scan.ingress_module.handle_event(httpx_event)
-    assert result is None
-    assert "httpx-only" in httpx_event.tags
     await scan._cleanup()

bbot/test/test_step_2/module_tests/base.py CHANGED Viewed

@@ -61,6 +61,7 @@ class ModuleTestBase:
                 config=self.config,
                 whitelist=module_test_base.whitelist,
                 blacklist=module_test_base.blacklist,
+                force_start=getattr(module_test_base, "force_start", False),
             )
             self.events = []
             self.log = logging.getLogger(f"bbot.test.{module_test_base.name}")
@@ -108,10 +109,14 @@ class ModuleTestBase:
         self.log.debug("Executing setup_after_prep()")
         await self.setup_after_prep(module_test)
         self.log.debug("Starting scan")
-        module_test.events = [e async for e in module_test.scan.async_start()]
+        await self._execute_scan(module_test)
         self.log.debug(f"Finished {module_test.name} module test")
         yield module_test
+    async def _execute_scan(self, module_test):
+        """Execute the scan and collect events. Can be overridden by benchmark classes."""
+        module_test.events = [e async for e in module_test.scan.async_start()]
     @pytest.mark.asyncio
     async def test_module_run(self, module_test):
         from bbot.core.helpers.misc import execute_sync_or_async

bbot/test/test_step_2/module_tests/test_module_dnsbimi.py CHANGED Viewed

@@ -6,11 +6,12 @@ raw_bimi_txt_default = (
 raw_bimi_txt_nondefault = '"v=BIMI1; l=https://nondefault.thirdparty.tld/brand/logo.svg;a=https://nondefault.thirdparty.tld/brand/certificate.pem;"'
-class TestBIMI(ModuleTestBase):
+class TestDnsbimi(ModuleTestBase):
     targets = ["test.localdomain"]
     modules_overrides = ["dnsbimi", "speculate"]
     config_overrides = {
         "modules": {"dnsbimi": {"emit_raw_dns_records": True, "selectors": "default,nondefault"}},
+        "omit_event_types": ["HTTP_RESPONSE", "RAW_TEXT", "DNS_NAME_UNRESOLVED", "FILESYSTEM", "WEB_PARAMETER"],
     }
     async def setup_after_prep(self, module_test):

bbot 2.6.0.6879rc0__py3-none-any.whl → 2.7.2.7254rc0__py3-none-any.whl

Potentially problematic release.

bbot 2.6.0.6879rc0py3-none-any.whl → 2.7.2.7254rc0py3-none-any.whl