summit-scanner 0.6.7__py3-none-any.whl

summit/__init__.py

@@ -0,0 +1,3 @@
+"""Summit - open-source web security testing platform."""
+
+__version__ = "0.6.0"

summit/authorize.py

@@ -0,0 +1,137 @@
+"""Authorization testing - the Autorize workflow, made scriptable.
+
+You provide URLs and two role definitions - typically a privileged
+session and a low-privilege one. Summit requests every URL as every
+role, compares the responses, and flags the case that matters most:
+a low-privilege role receiving the same answer as the privileged one.
+That is broken access control (BOLA/IDOR) - the single most
+consequential bug class in bug bounty.
+
+Verdicts per URL:
+
+- ``enforced`` - the roles see different responses (working authz)
+- ``broken``   - the low-privilege response matches the privileged one
+- ``partial``  - same status, different body (worth a human look)
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+from .http.client import HttpClient
+from .http.models import HttpRequest, HttpResponse
+from .scan.core import Finding
+
+
+@dataclass
+class Role:
+    name: str
+    headers: dict[str, str]
+
+
+@dataclass
+class AuthResult:
+    url: str
+    privileged_role: str
+    other_role: str
+    privileged: HttpResponse
+    other: HttpResponse
+    verdict: str
+
+
+def _signature(response: HttpResponse) -> tuple:
+    return (response.status, len(response.body), hashlib.sha256(response.body).hexdigest()[:16])
+
+
+def compare(privileged: HttpResponse, other: HttpResponse) -> str:
+    if _signature(privileged) == _signature(other):
+        return "broken"
+    if privileged.status == other.status:
+        return "partial"
+    return "enforced"
+
+
+def run_authorize(
+    client: HttpClient,
+    urls: list[str],
+    privileged: Role,
+    other: Role,
+) -> list[AuthResult]:
+    """Request every URL as both roles and classify the difference."""
+    results: list[AuthResult] = []
+    for url in urls:
+        try:
+            privileged_response = client.send(HttpRequest(method="GET", url=url, headers=dict(privileged.headers)))
+            other_response = client.send(HttpRequest(method="GET", url=url, headers=dict(other.headers)))
+        except Exception:
+            continue  # unreachable URL - report nothing, the scanner already handles reachability
+        results.append(
+            AuthResult(
+                url=url,
+                privileged_role=privileged.name,
+                other_role=other.name,
+                privileged=privileged_response,
+                other=other_response,
+                verdict=compare(privileged_response, other_response),
+            )
+        )
+    return results
+
+
+def findings_from(results: list[AuthResult]) -> list[Finding]:
+    findings: list[Finding] = []
+    for result in results:
+        if result.verdict == "broken":
+            # Guard against the classic login-redirect false positive: if the
+            # "privileged" response is itself a redirect or an empty shell,
+            # both roles are probably bouncing off the same auth wall - there
+            # is no privileged content to leak in the first place.
+            privileged_usable = (
+                result.privileged.status == 200
+                and len(result.privileged.body) >= 100
+            )
+            if not privileged_usable:
+                continue
+            findings.append(
+                Finding(
+                    check_id="authorize-bypass",
+                    title="Broken access control (BOLA/IDOR candidate)",
+                    severity="critical",
+                    url=result.url,
+                    confidence="confirmed",
+                    detail=(
+                        f"Role '{result.other_role}' received an identical response to privileged role "
+                        f"'{result.privileged_role}' (same status, same body). The endpoint does not "
+                        "enforce authorization - verify the object ids are truly equivalent for both roles."
+                    ),
+                    evidence=f"both roles: HTTP {result.privileged.status}, {len(result.privileged.body)} bytes",
+                    remediation="Enforce authorization server-side on every request; deny by default.",
+                )
+            )
+        elif result.verdict == "partial":
+            findings.append(
+                Finding(
+                    check_id="authorize-partial",
+                    title="Authorization response differs only in body",
+                    severity="medium",
+                    url=result.url,
+                    confidence="potential",
+                    detail=(
+                        f"Roles '{result.privileged_role}' and '{result.other_role}' get the same status "
+                        "but different bodies - could be personalization, could be leaking privileged data. Worth a human look."
+                    ),
+                    evidence=f"HTTP {result.privileged.status}, privileged {len(result.privileged.body)}B vs other {len(result.other.body)}B",
+                    remediation="Confirm the response difference is intentional personalization, not data leakage.",
+                )
+            )
+    return findings
+
+
+def parse_role(spec: str) -> Role:
+    """Parse 'name=Header: value' into a Role, e.g. 'admin=Cookie: session=abc'."""
+    name, _, rest = spec.partition("=")
+    header_name, _, header_value = rest.partition(":")
+    if not name.strip() or not header_name.strip():
+        raise ValueError(f"role spec must look like 'name=Header: value', got: {spec!r}")
+    return Role(name.strip(), {header_name.strip(): header_value.strip()})

summit/captcha.py

@@ -0,0 +1,55 @@
+"""CAPTCHA and anti-bot challenge detection.
+
+Summit does not solve CAPTCHAs - that belongs to a human with a browser,
+and automating it would cross lines we will not cross. What the scanner
+does is detect the challenge, tell you exactly which one is in the way,
+and stop burning requests against it. Knowing "this endpoint sits behind
+reCAPTCHA" is actionable; hammering it is not.
+"""
+
+from __future__ import annotations
+
+from .http.models import HttpResponse
+from .scan.core import Finding
+
+CHALLENGE_SIGNATURES: list[tuple[str, list[str]]] = [
+    ("reCAPTCHA", ["recaptcha/api.js", "g-recaptcha", "grecaptcha.execute"]),
+    ("hCaptcha", ["hcaptcha.com/1/api.js", "h-captcha"]),
+    ("Cloudflare challenge", ["cf_chl_opt", "checking your browser", "verifying you are human"]),
+]
+
+_TURNSTILE = "challenges.cloudflare.com/turnstile"
+_MAX_BODY = 20_000
+
+
+def detect_challenge(response: HttpResponse) -> tuple[str, str] | None:
+    """Return (challenge name, evidence) when the response contains one."""
+    body = response.text[:_MAX_BODY].lower()
+    if _TURNSTILE in body:
+        return "Cloudflare Turnstile", _TURNSTILE
+    for name, markers in CHALLENGE_SIGNATURES:
+        for marker in markers:
+            if marker in body:
+                return name, marker
+    return None
+
+
+def check_challenge(response: HttpResponse) -> list[Finding]:
+    detected = detect_challenge(response)
+    if not detected:
+        return []
+    name, evidence = detected
+    return [
+        Finding(
+            check_id="captcha-challenge",
+            title=f"CAPTCHA/challenge detected: {name}",
+            severity="info",
+            url=response.request.url,
+            detail=(
+                f"This endpoint is protected by {name}. Summit reports the challenge "
+                "instead of hammering it - automated CAPTCHA solving is out of scope "
+                "by design. Test this surface manually in a browser."
+            ),
+            evidence=evidence,
+        )
+    ]

summit/cli.py

@@ -0,0 +1,366 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from . import __version__
+from .authorize import findings_from, parse_role, run_authorize
+from .crawl.spider import Spider
+from .fuzz.dirbuster import fuzz
+from .http.client import HttpClient
+from .http.models import HttpRequest
+from .openapi import load_spec, spec_pages
+from .paramminer import mine
+from .proxy.server import ProxyServer
+from .recon import missing_tools, render_report, run_recon
+from .report.writer import write_console, write_html, write_json
+from .scan.core import SEVERITIES, filter_by_min_severity, filter_by_severities
+from .scan.engine import ScanEngine
+from .toolkit import decode_text, encode_text
+from .tls import audit as tls_audit
+from .updates import fetch_latest_version, install_tools, is_outdated, self_update
+
+BANNER = r"""
+          _
+ ___ _ _| |_ ___ _ _ ___  _ _ ___ ___
+|  _| ' \  _/ -_) '_/ -_) '_\ \ / / -_)   summit v{version}
+|___|_||_\__\___|_| \___|_| /_\_\ \___|   by Pennacle Security
+                                          authorized testing only
+"""
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="summit",
+        description="Summit - open-source web security testing platform (proxy, scanner, fuzzer, link hijack detection).",
+    )
+    parser.add_argument("--version", action="version", version=f"summit {__version__}")
+    parser.add_argument("--check-update", action="store_true", help="compare the installed version against the latest GitHub release")
+    parser.add_argument("--auto-update", action="store_true", help="update summit in place (git checkout) or print exact install steps")
+    parser.add_argument("--install-tools", action="store_true", help="install recon tools (nmap, httpx, subfinder) via the available package manager")
+    sub = parser.add_subparsers(dest="command")
+
+    scan = sub.add_parser("scan", help="crawl a target and run passive + active checks")
+    scan.add_argument("url", help="target URL, e.g. https://testsite.local")
+    scan.add_argument("--openapi", dest="openapi_path", help="scan an OpenAPI/JSON spec instead of crawling (paths become targets)")
+    scan.add_argument("--depth", type=int, default=3, help="max crawl depth (default 3)")
+    scan.add_argument("--max-pages", type=int, default=50, help="max pages to visit (default 50)")
+    scan.add_argument("--no-active", action="store_true", help="skip active checks (passive only)")
+    scan.add_argument("--skip-linkcheck", action="store_true", help="skip broken link / hijack check")
+    scan.add_argument("--workers", type=int, default=8, help="parallel check workers (default 8)")
+    scan.add_argument("--rate", type=float, default=None, help="max requests per second across all workers, e.g. --rate 5")
+    scan.add_argument("--retries", type=int, default=1, help="retries on connection errors and 429/503 (default 1)")
+    scan.add_argument("--delay", type=float, default=0.0, help="seconds between requests (be nice)")
+    scan.add_argument("--timeout", type=float, default=10.0, help="request timeout in seconds")
+    scan.add_argument("--json", dest="json_path", help="write a JSON report to this path")
+    scan.add_argument("--html", dest="html_path", help="write an HTML report to this path")
+    scan.add_argument("--severity", default=None, help="only report these severities, comma separated: high,critical")
+    scan.add_argument("--min-severity", dest="min_severity", default=None,
+                      choices=list(SEVERITIES), help="report findings at or above this severity")
+
+    mine_p = sub.add_parser("mine", help="discover hidden parameters on one URL (Param Miner approach)")
+    mine_p.add_argument("url")
+    mine_p.add_argument("--workers", type=int, default=8)
+    mine_p.add_argument("--timeout", type=float, default=10.0)
+    mine_p.add_argument("--json", dest="json_path", help="write findings as JSON")
+
+    auth = sub.add_parser("authorize", help="compare two roles per URL - broken access control (Autorize workflow)")
+    auth.add_argument("urls", nargs="+", help="URLs to test (test each with both roles)")
+    auth.add_argument("--role", dest="roles", action="append", required=True,
+                      help="role spec 'name=Header: value', repeatable; first role = privileged baseline, e.g. --role 'admin=Cookie: session=abc'")
+    auth.add_argument("--timeout", type=float, default=10.0)
+    auth.add_argument("--json", dest="json_path", help="write findings as JSON")
+
+    fuzz_p = sub.add_parser("fuzz", help="brute-force paths on a target")
+    fuzz_p.add_argument("url")
+    fuzz_p.add_argument("-w", "--wordlist", help="wordlist file, one entry per line")
+    fuzz_p.add_argument("-t", "--threads", type=int, default=10, help="concurrent requests (default 10)")
+    fuzz_p.add_argument("-e", "--extensions", default="", help="comma separated, e.g. php,html,bak")
+    fuzz_p.add_argument("--rate", type=float, default=None, help="max requests per second")
+    fuzz_p.add_argument("--timeout", type=float, default=10.0)
+
+    proxy_p = sub.add_parser("proxy", help="run the local intercepting proxy")
+    proxy_p.add_argument("--bind", default="127.0.0.1")
+    proxy_p.add_argument("--port", type=int, default=8080)
+    proxy_p.add_argument("--db", default="summit_sitemap.db", help="sqlite site map path")
+
+    rep = sub.add_parser("repeater", help="send a raw HTTP request from a file (Burp-style)")
+    rep.add_argument("file")
+    rep.add_argument("--timeout", type=float, default=10.0)
+
+    rec = sub.add_parser("recon", help="chain subfinder -> httpx -> nmap on a target")
+    rec.add_argument("target", help="domain or URL, e.g. testsite.local")
+    rec.add_argument("--timeout", type=float, default=120.0, help="per-tool timeout in seconds")
+    rec.add_argument("--json", dest="json_path", help="write the recon report as JSON")
+
+    enc = sub.add_parser("encode", help="encode text (b64, url, hex)")
+    enc.add_argument("value")
+    enc.add_argument("-f", "--format", dest="encoding", default="b64", choices=["b64", "url", "hex"])
+
+    dec = sub.add_parser("decode", help="decode text (b64, url, hex, jwt)")
+    dec.add_argument("value")
+    dec.add_argument("-f", "--format", dest="encoding", default="b64", choices=["b64", "url", "hex", "jwt"])
+
+    tls_p = sub.add_parser("tls", help="audit a host's certificate and TLS versions")
+    tls_p.add_argument("host", help="hostname or URL")
+    tls_p.add_argument("--port", type=int, default=443)
+    tls_p.add_argument("--timeout", type=float, default=10.0)
+    tls_p.add_argument("--json", dest="json_path", help="write findings as JSON")
+
+    return parser
+
+
+def _cmd_check_update() -> int:
+    print(BANNER.format(version=__version__))
+    latest = fetch_latest_version()
+    if latest is None:
+        print("[!] could not reach the GitHub release API (offline or rate-limited)")
+        return 1
+    if is_outdated(__version__, latest):
+        print(f"[*] update available: {__version__} -> {latest}")
+        print("[*] run: summit --auto-update")
+    else:
+        print(f"[*] summit {__version__} is up to date (latest release: {latest})")
+    return 0
+
+
+def _cmd_auto_update() -> int:
+    print(BANNER.format(version=__version__))
+    ok, message = self_update()
+    print(f"[*] {message}" if ok else f"[!] {message}")
+    return 0 if ok else 1
+
+
+def _cmd_install_tools() -> int:
+    print(BANNER.format(version=__version__))
+    install_tools()
+    return 0
+
+
+def _severity_filters(args: argparse.Namespace):
+    """Parse --severity / --min-severity into (allowed set, floor name)."""
+    allowed = None
+    if getattr(args, "severity", None):
+        allowed = {token.strip().lower() for token in args.severity.split(",") if token.strip()}
+        unknown = allowed - set(SEVERITIES)
+        if unknown:
+            print(f"[!] unknown severity value(s): {', '.join(sorted(unknown))} - valid: {', '.join(SEVERITIES)}")
+            raise SystemExit(1)
+    return allowed, getattr(args, "min_severity", None)
+
+
+def _apply_severity_filters(findings, args: argparse.Namespace):
+    allowed, minimum = _severity_filters(args)
+    if allowed:
+        findings = filter_by_severities(findings, allowed)
+    if minimum:
+        findings = filter_by_min_severity(findings, minimum)
+    return findings
+
+
+def _cmd_scan(args: argparse.Namespace) -> int:
+    print(BANNER.format(version=__version__))
+    client = HttpClient(timeout=args.timeout, delay=args.delay, rate=args.rate, retries=args.retries)
+    if args.openapi_path:
+        spec = load_spec(args.openapi_path)
+        result = spec_pages(client, spec)
+        print(f"[*] openapi spec loaded: {len(result['pages'])} target(s), {len(result['errors'])} error(s)")
+    else:
+        spider = Spider(client, max_depth=args.depth, max_pages=args.max_pages, delay=args.delay)
+        print(f"[*] crawling {args.url} ...")
+        result = spider.crawl(args.url)
+        print(
+            f"[*] visited {len(result['visited'])} page(s), "
+            f"{len(result['forms'])} form(s), {len(result['external_links'])} outbound link(s), "
+            f"{len(result['errors'])} request error(s)"
+        )
+    for error in result["errors"][:5]:
+        print(f"[!] {error}")
+
+    engine = ScanEngine(
+        client,
+        active=not args.no_active,
+        workers=args.workers,
+        linkcheck_enabled=not args.skip_linkcheck,
+    )
+    findings = engine.scan(result)
+    findings = _apply_severity_filters(findings, args)
+    print(
+        f"[*] {client.request_count} request(s) in {engine.elapsed:.1f}s "
+        f"(workers={args.workers}, rate={args.rate or 'unlimited'}, retries={args.retries})"
+    )
+    print(f"[*] smart order: {engine.high_signal} high-signal injection point(s) probed first")
+    for error in engine.errors[:5]:
+        print(f"[!] check error: {error}")
+    print(write_console(findings, args.url))
+    if args.json_path:
+        write_json(findings, args.json_path)
+        print(f"[*] JSON report written to {args.json_path}")
+    if args.html_path:
+        write_html(findings, args.html_path, args.url)
+        print(f"[*] HTML report written to {args.html_path}")
+    return 0
+
+
+def _cmd_fuzz(args: argparse.Namespace) -> int:
+    wordlist = None
+    if args.wordlist:
+        wordlist = [line.strip() for line in Path(args.wordlist).read_text(encoding="utf-8").splitlines() if line.strip()]
+    extensions = [ext.strip().lstrip(".") for ext in args.extensions.split(",") if ext.strip()]
+    client = HttpClient(timeout=args.timeout, rate=args.rate)
+    findings = fuzz(client, args.url, wordlist=wordlist, threads=args.threads, extensions=extensions)
+    print(write_console(findings, args.url))
+    return 0
+
+
+def _cmd_proxy(args: argparse.Namespace) -> int:
+    server = ProxyServer(host=args.bind, port=args.port, db_path=args.db)
+    try:
+        server.start()
+    except KeyboardInterrupt:
+        print("\n[*] shutting down")
+        server.stop()
+    return 0
+
+
+def _cmd_repeater(args: argparse.Namespace) -> int:
+    raw = Path(args.file).read_text(encoding="utf-8")
+    request = HttpRequest.from_raw(raw)
+    client = HttpClient(timeout=args.timeout)
+    print(f"[*] sending {request.method} {request.url}")
+    response = client.send(request)
+    print(f"HTTP {response.status} ({response.elapsed:.2f}s)")
+    for key, value in response.headers.items():
+        print(f"{key}: {value}")
+    print()
+    print(response.text[:5000])
+    return 0
+
+
+def _cmd_mine(args: argparse.Namespace) -> int:
+    print(BANNER.format(version=__version__))
+    client = HttpClient(timeout=args.timeout)
+    print(f"[*] mining hidden parameters on {args.url} ...")
+    findings = mine(client, args.url, workers=args.workers)
+    print(write_console(findings, args.url))
+    if args.json_path:
+        write_json(findings, args.json_path)
+        print(f"[*] JSON report written to {args.json_path}")
+    return 0
+
+
+def _cmd_authorize(args: argparse.Namespace) -> int:
+    print(BANNER.format(version=__version__))
+    client = HttpClient(timeout=args.timeout)
+    try:
+        roles = [parse_role(spec) for spec in args.roles]
+    except ValueError as exc:
+        print(f"[!] {exc}")
+        return 1
+    if len(roles) < 2:
+        print("[!] authorize needs at least two roles - first is the privileged baseline")
+        return 1
+    privileged, other = roles[0], roles[1]
+    print(f"[*] testing {len(args.urls)} URL(s): '{privileged.name}' vs '{other.name}'")
+    results = run_authorize(client, args.urls, privileged, other)
+    for result in results:
+        print(f"  [{result.verdict.upper()}] {result.url}")
+        if result.verdict == "broken":
+            print(f"      {other.name} sees exactly what {privileged.name} sees (HTTP {result.privileged.status}, {len(result.privileged.body)}B)")
+    findings = findings_from(results)
+    print(write_console(findings, f"{len(args.urls)} URL(s), {privileged.name} vs {other.name}"))
+    if args.json_path:
+        write_json(findings, args.json_path)
+        print(f"[*] JSON report written to {args.json_path}")
+    return 0
+
+
+def _cmd_recon(args: argparse.Namespace) -> int:
+    print(BANNER.format(version=__version__))
+    missing = missing_tools()
+    if missing:
+        print(f"[!] tools not on PATH: {', '.join(missing)} - install them with: summit --install-tools")
+        print("[*] continuing with the tools that ARE available ...\n")
+    try:
+        report = run_recon(args.target, timeout=args.timeout)
+    except Exception as exc:
+        print(f"[!] recon failed: {exc}")
+        return 1
+    print(render_report(report))
+    if args.json_path:
+        Path(args.json_path).write_text(json.dumps(report, indent=2), encoding="utf-8")
+        print(f"\n[*] JSON report written to {args.json_path}")
+    return 0
+
+
+def _cmd_tls(args: argparse.Namespace) -> int:
+    print(BANNER.format(version=__version__))
+    try:
+        findings = tls_audit(args.host, port=args.port, timeout=args.timeout)
+    except ConnectionError as exc:
+        print(f"[!] {exc}")
+        return 1
+    print(write_console(findings, f"{args.host}:{args.port}"))
+    if not findings:
+        print("[*] no TLS findings - certificate valid, no legacy protocol versions accepted")
+    if args.json_path:
+        write_json(findings, args.json_path)
+        print(f"[*] JSON report written to {args.json_path}")
+    return 0
+
+
+def _cmd_encode(args: argparse.Namespace) -> int:
+    try:
+        print(encode_text(args.value, args.encoding))
+    except ValueError as exc:
+        print(f"[!] {exc}")
+        return 1
+    return 0
+
+
+def _cmd_decode(args: argparse.Namespace) -> int:
+    try:
+        print(decode_text(args.value, args.encoding))
+    except ValueError as exc:
+        print(f"[!] {exc}")
+        return 1
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        if args.check_update:
+            return _cmd_check_update()
+        if args.auto_update:
+            return _cmd_auto_update()
+        if args.install_tools:
+            return _cmd_install_tools()
+        if args.command == "scan":
+            return _cmd_scan(args)
+        if args.command == "fuzz":
+            return _cmd_fuzz(args)
+        if args.command == "proxy":
+            return _cmd_proxy(args)
+        if args.command == "repeater":
+            return _cmd_repeater(args)
+        if args.command == "mine":
+            return _cmd_mine(args)
+        if args.command == "authorize":
+            return _cmd_authorize(args)
+        if args.command == "recon":
+            return _cmd_recon(args)
+        if args.command == "tls":
+            return _cmd_tls(args)
+        if args.command == "encode":
+            return _cmd_encode(args)
+        if args.command == "decode":
+            return _cmd_decode(args)
+    except KeyboardInterrupt:
+        print("\n[!] interrupted")
+        return 130
+    print(f"summit {__version__}")
+    parser.print_help()
+    return 0

summit/crawl/js_endpoints.py

@@ -0,0 +1,57 @@
+"""JavaScript endpoint extraction - the AJAX-spider idea, stdlib edition.
+
+Modern applications hide most of their attack surface inside JavaScript:
+fetch() calls, axios calls, XMLHttpRequest targets. A crawler that only
+follows <a href> sees none of it - a systematic false-negative source.
+
+This module regex-extracts endpoint URLs from every fetched page (and
+from .js files it references), resolves them against the page URL, and
+hands same-host matches back to the spider queue.
+
+Regexes are anchored on the calling APIs themselves (fetch/axios/XHR),
+not on generic string hunting, which keeps the hit rate honest.
+"""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import urljoin
+
+# endpoint-extraction patterns; group 1 must be the endpoint
+JS_PATTERNS = [
+    re.compile(r"""fetch\(\s*['"]([^'"]+)['"]"""),
+    re.compile(r"""axios(?:\.\w+)?\(\s*['"]([^'"]+)['"]"""),
+    re.compile(r"""axios\.\w+\(\s*['"]([^'"]+)['"]"""),
+    re.compile(r"""\.open\(\s*['"](?:[A-Z]+)['"],\s*['"]([^'"]+)['"]"""),
+]
+
+JS_FILE_PATTERN = re.compile(r"""(?:src|href)\s*=\s*['"]([^'"]+\.js(?:\?[^'"]*)?)['"]""", re.IGNORECASE)
+
+
+def extract_endpoints(text: str, base_url: str) -> tuple[list[str], list[str]]:
+    """Return (endpoints, js_files) found in one document, resolved absolute."""
+    endpoints: list[str] = []
+    js_files: list[str] = []
+    for pattern in JS_PATTERNS:
+        for match in pattern.finditer(text):
+            endpoints.append(urljoin(base_url, match.group(1)))
+    for match in JS_FILE_PATTERN.finditer(text):
+        js_files.append(urljoin(base_url, match.group(1)))
+    return endpoints, js_files
+
+
+def extract_from_page(response_text: str, page_url: str, scope_hosts: set[str]) -> tuple[list[str], list[str]]:
+    """Extract endpoints from a page, then from any same-host .js file it references.
+
+    Returns (same_host_endpoints, external_endpoints). Fetching the .js
+    files is the caller's job (it has the HTTP client).
+    """
+    endpoints, js_files = extract_endpoints(response_text, page_url)
+    return endpoints, js_files
+
+
+def in_scope(url: str, scope_hosts: set[str]) -> bool:
+    from urllib.parse import urlsplit
+
+    host = (urlsplit(url).hostname or "").lower().removeprefix("www.")
+    return any(host == candidate or host.endswith("." + candidate) for candidate in scope_hosts)