blackops-core 0.1.5__py3-none-any.whl

blackops_core/__init__.py

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (c) 2026 CommonHuman-Lab
+"""
+blackops-core — shared HTTP, crawling, and scanning infrastructure.
+"""
+
+__version__ = "0.1.5"
+
+__all__ = ["__version__"]

blackops_core/auth.py

@@ -0,0 +1,241 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (c) 2026 CommonHuman-Lab
+"""
+blackops-core — auth.py
+Form-based and token-based authentication helpers.
+
+Provides:
+  - form_login()   — POST credentials to an HTML login form, return session cookies/headers
+  - bearer_login() — OAuth 2.0 client credentials grant, return Authorization header
+  - extract_csrf() — pull a CSRF token from an HTML page
+  - AuthResult     — carries cookies (str) + headers (dict) ready for subsequent requests
+"""
+from __future__ import annotations
+
+import json
+import logging
+import urllib.parse as up
+from dataclasses import dataclass, field
+from html.parser import HTMLParser
+from typing import Any, Dict, List, Optional
+
+from .http.client import HttpClient
+
+logger = logging.getLogger(__name__)
+
+_CSRF_NAMES = frozenset({
+    "csrf_token", "_token", "xsrf_token", "authenticity_token",
+    "csrfmiddlewaretoken", "_csrf", "csrf", "__requestverificationtoken",
+})
+
+
+# ---------------------------------------------------------------------------
+# Public data type
+# ---------------------------------------------------------------------------
+
+@dataclass
+class AuthResult:
+    """Session credentials ready to pass to any CommonHuman-Lab scanner."""
+    cookies: str = ""
+    headers: Dict[str, str] = field(default_factory=dict)
+
+    def is_empty(self) -> bool:
+        return not self.cookies and not self.headers
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def form_login(
+    login_url:      str,
+    username:       str,
+    password:       str,
+    username_field: str = "username",
+    password_field: str = "password",
+    extra_fields:   Optional[Dict[str, str]] = None,
+    client:         Optional[HttpClient] = None,
+    timeout:        int = 15,
+) -> AuthResult:
+    """Submit an HTML login form and return the resulting session.
+
+    Fetches the login page, extracts CSRF tokens and hidden fields, POSTs
+    credentials to the form action, then collects session cookies.  If the
+    server responds with a JSON body containing a token field the Bearer
+    header is populated automatically.
+
+    Returns an empty AuthResult on network or parse failure.
+    """
+    c = client or HttpClient(timeout=timeout)
+
+    try:
+        resp = c._session.get(login_url, timeout=timeout)
+    except Exception as exc:
+        logger.warning("form_login: GET %s failed: %s", login_url, exc)
+        return AuthResult()
+
+    parser = _FormParser()
+    parser.feed(resp.text)
+
+    action = parser.action or login_url
+    if action and not action.startswith(("http://", "https://")):
+        action = up.urljoin(login_url, action)
+
+    body: Dict[str, str] = dict(parser.fields)
+    body[username_field] = username
+    body[password_field] = password
+    if extra_fields:
+        body.update(extra_fields)
+
+    try:
+        post_resp = c._session.post(action, data=body, timeout=timeout, allow_redirects=True)
+    except Exception as exc:
+        logger.warning("form_login: POST %s failed: %s", action, exc)
+        return AuthResult()
+
+    cookies = "; ".join(
+        f"{name}={val}"
+        for name, val in c._session.cookies.items()
+    )
+
+    result_headers: Dict[str, str] = {}
+    try:
+        j = post_resp.json()
+        for key in ("token", "access_token", "accessToken", "jwt", "id_token"):
+            if key in j and isinstance(j[key], str):
+                result_headers["Authorization"] = f"Bearer {j[key]}"
+                break
+    except Exception:
+        pass
+
+    result = AuthResult(cookies=cookies, headers=result_headers)
+    if result.is_empty():
+        logger.warning("form_login: no cookies or token obtained from %s", login_url)
+    else:
+        logger.info("form_login: authenticated via %s (%d cookies)", login_url, cookies.count(";") + 1)
+    return result
+
+
+def bearer_login(
+    token_url:     str,
+    client_id:     str,
+    client_secret: str,
+    grant_type:    str = "client_credentials",
+    client:        Optional[HttpClient] = None,
+    timeout:       int = 15,
+) -> AuthResult:
+    """OAuth 2.0 token endpoint — client credentials or password grant.
+
+    Returns AuthResult with Authorization: Bearer <token> header populated,
+    or empty AuthResult on failure.
+    """
+    c = client or HttpClient(timeout=timeout)
+    body = {
+        "grant_type":    grant_type,
+        "client_id":     client_id,
+        "client_secret": client_secret,
+    }
+    try:
+        resp = c._session.post(token_url, data=body, timeout=timeout)
+        j = resp.json()
+        token = j.get("access_token") or j.get("token") or j.get("id_token")
+        if token:
+            logger.info("bearer_login: obtained token from %s", token_url)
+            return AuthResult(headers={"Authorization": f"Bearer {token}"})
+    except Exception as exc:
+        logger.warning("bearer_login: %s failed: %s", token_url, exc)
+    return AuthResult()
+
+
+def http_auth(auth_type: str, cred: str) -> Any:
+    """Return a requests-compatible auth object for Basic, Digest, or NTLM auth.
+
+    Args:
+        auth_type: ``"basic"``, ``"digest"``, or ``"ntlm"``.
+        cred:      Credentials in ``"username:password"`` format. The password
+                   may itself contain colons — only the first colon is used as
+                   the delimiter.
+
+    Returns:
+        A ``requests.auth.HTTPBasicAuth``, ``requests.auth.HTTPDigestAuth``,
+        or ``requests_ntlm.HttpNtlmAuth`` instance ready to be passed to
+        ``HttpClient(auth=...)``.
+
+    Raises:
+        ValueError:  Invalid *auth_type* or malformed *cred*.
+        ImportError: ``auth_type="ntlm"`` requested but ``requests-ntlm`` is
+                     not installed (``pip install blackops-core[ntlm]``).
+    """
+    if not cred or ":" not in cred:
+        raise ValueError(
+            f"auth_cred must be in 'username:password' format, got {cred!r}"
+        )
+    user, _, password = cred.partition(":")
+
+    if auth_type == "basic":
+        from requests.auth import HTTPBasicAuth
+        return HTTPBasicAuth(user, password)
+    if auth_type == "digest":
+        from requests.auth import HTTPDigestAuth
+        return HTTPDigestAuth(user, password)
+    if auth_type == "ntlm":
+        try:
+            from requests_ntlm import HttpNtlmAuth  # type: ignore[import]
+        except ImportError as exc:
+            raise ImportError(
+                "NTLM auth requires requests-ntlm: pip install blackops-core[ntlm]"
+            ) from exc
+        return HttpNtlmAuth(user, password)
+    raise ValueError(
+        f"Unknown auth_type {auth_type!r}. Supported values: basic, digest, ntlm"
+    )
+
+
+def extract_csrf(html: str) -> Optional[str]:
+    """Extract a CSRF token from an HTML page.
+
+    Scans for ``<input type="hidden">`` elements whose name matches known
+    CSRF field name patterns.  Returns the first value found, or None.
+    """
+    parser = _FormParser()
+    parser.feed(html)
+    for name, value in parser.fields.items():
+        if name.lower() in _CSRF_NAMES and value:
+            return value
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Internal HTML form parser
+# ---------------------------------------------------------------------------
+
+class _FormParser(HTMLParser):
+    """Minimal parser that extracts the first HTML form's action and fields."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.action: str = ""
+        self.method: str = "post"
+        self.fields: Dict[str, str] = {}
+        self._in_form: bool = False
+        self._done: bool = False
+
+    def handle_starttag(self, tag: str, attrs: List) -> None:
+        if self._done:
+            return
+        a = dict(attrs)
+        if tag == "form" and not self._in_form:
+            self._in_form = True
+            self.action = a.get("action", "")
+            self.method = a.get("method", "post").lower()
+        elif tag == "input" and self._in_form:
+            name  = a.get("name", "")
+            value = a.get("value") or ""
+            itype = a.get("type", "text").lower()
+            if name and itype not in ("submit", "button", "image", "reset", "file"):
+                self.fields[name] = value
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag == "form" and self._in_form:
+            self._in_form = False
+            self._done = True

blackops_core/browser_crawler.py

@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (c) 2026 CommonHuman-Lab
+"""
+blackops-core — browser_crawler.py
+Headless Chromium-based URL discovery for JavaScript-rendered sites.
+
+Unlike the standard BFS crawler (which parses static HTML), this module
+renders each page with Selenium, waits for JavaScript to complete, and
+collects all links present in the fully-rendered DOM.  Same-origin only.
+
+Requires: selenium>=4.0
+    pip install 'blackops-core[browser]'
+"""
+from __future__ import annotations
+
+import logging
+import time
+import urllib.parse as up
+from collections import deque
+from typing import Dict, List, Optional, Set
+
+logger = logging.getLogger(__name__)
+
+_WAIT_FIRST_PAGE = 1.0   # max seconds to wait for first page readyState=complete
+_WAIT_SUBSEQUENT = 0.75  # max seconds to wait for subsequent pages
+_SPA_SETTLE      = 0.25  # fixed pause after readyState=complete for SPA first render
+_POLL_INTERVAL   = 0.05  # polling granularity in seconds
+
+
+def _wait_for_ready(driver, timeout_s: float) -> None:
+    """Poll until document.readyState == 'complete' or timeout elapses, then settle."""
+    deadline = time.monotonic() + timeout_s
+    while True:
+        try:
+            if driver.execute_script("return document.readyState") == "complete":
+                break
+        except Exception:
+            pass
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            break
+        time.sleep(min(_POLL_INTERVAL, remaining))
+    time.sleep(_SPA_SETTLE)
+
+
+def browser_crawl(
+    start_url:         str,
+    max_pages:         int = 50,
+    max_depth:         int = 2,
+    headless:          bool = True,
+    cookies:           str = "",
+    extra_headers:     Optional[Dict[str, str]] = None,
+    chromium_path:     str = "",
+    chromedriver_path: str = "",
+    spa_wait_s:        float = _WAIT_SUBSEQUENT,
+) -> List[str]:
+    """Discover URLs by rendering pages with headless Chromium.
+
+    Performs BFS from ``start_url``, rendering each page with Selenium and
+    collecting ``<a href>`` links from the fully-rendered DOM.  Only follows
+    same-origin URLs.  Returns a deduplicated list of visited URLs.
+
+    Parameters
+    ----------
+    start_url:
+        Seed URL to start from.
+    max_pages:
+        Stop after visiting this many unique pages.
+    max_depth:
+        Maximum BFS depth from start_url.
+    headless:
+        Run Chromium without a visible window (default True).
+    cookies:
+        Cookie string injected before the first request (``name=val; name2=val2``).
+    extra_headers:
+        Not injected at the driver level (Selenium has limited header support);
+        reserved for future CDP-based header injection.
+    chromium_path:
+        Path to Chromium binary.  Auto-detected if empty.
+    chromedriver_path:
+        Path to chromedriver binary.  Auto-detected if empty.
+    spa_wait_s:
+        Seconds to wait after each page navigation for JS to render.
+    """
+    try:
+        driver = _setup_driver(headless, chromium_path, chromedriver_path)
+    except ImportError as exc:
+        logger.error(
+            "browser_crawl requires selenium — pip install 'blackops-core[browser]'. %s", exc
+        )
+        return []
+    except Exception as exc:
+        logger.error("browser_crawl: failed to start Chromium driver: %s", exc)
+        return []
+
+    parsed_start = up.urlparse(start_url)
+    origin       = f"{parsed_start.scheme}://{parsed_start.netloc}"
+
+    visited: List[str] = []
+    seen:    Set[str]  = set()
+    queue:   deque[tuple[str, int]] = deque([(start_url, 0)])
+    seen.add(_normalise(start_url))
+
+    try:
+        # Inject cookies on the origin before any navigation
+        if cookies:
+            try:
+                driver.get(origin)
+                _wait_for_ready(driver, 0.5)
+                for pair in cookies.split(";"):
+                    pair = pair.strip()
+                    if "=" in pair:
+                        name, _, value = pair.partition("=")
+                        driver.add_cookie({"name": name.strip(), "value": value.strip()})
+            except Exception as exc:
+                logger.debug("browser_crawl: cookie injection failed: %s", exc)
+
+        while queue and len(visited) < max_pages:
+            url, depth = queue.popleft()
+
+            try:
+                driver.get(url)
+                wait = _WAIT_FIRST_PAGE if depth == 0 else spa_wait_s
+                _wait_for_ready(driver, wait)
+            except Exception as exc:
+                logger.debug("browser_crawl: page load failed %s: %s", url, exc)
+                continue
+
+            visited.append(url)
+            logger.debug("browser_crawl: visited %s (depth=%d)", url, depth)
+
+            if depth >= max_depth:
+                continue
+
+            try:
+                links: List[str] = driver.execute_script(
+                    "return Array.from(document.querySelectorAll('a[href]'))"
+                    ".map(a => a.href)"
+                    ".filter(h => h.startsWith('http'));"
+                ) or []
+            except Exception:
+                links = []
+
+            for link in links:
+                norm = _normalise(link)
+                if norm in seen:
+                    continue
+                parsed = up.urlparse(link)
+                if f"{parsed.scheme}://{parsed.netloc}" != origin:
+                    continue
+                seen.add(norm)
+                queue.append((link, depth + 1))
+
+    except Exception as exc:
+        logger.warning("browser_crawl: unexpected error: %s", exc)
+    finally:
+        try:
+            driver.quit()
+        except Exception:
+            pass
+
+    logger.info("browser_crawl: discovered %d URL(s) from %s", len(visited), start_url)
+    return visited
+
+
+# ---------------------------------------------------------------------------
+# Selenium driver factory
+# ---------------------------------------------------------------------------
+
+def _setup_driver(headless: bool, chromium_path: str, chromedriver_path: str):
+    from selenium import webdriver  # noqa: PLC0415
+    from selenium.webdriver.chrome.options import Options  # noqa: PLC0415
+    from selenium.webdriver.chrome.service import Service  # noqa: PLC0415
+
+    opts = Options()
+    if headless:
+        opts.add_argument("--headless")
+    opts.add_argument("--no-sandbox")
+    opts.add_argument("--disable-dev-shm-usage")
+    opts.add_argument("--disable-gpu")
+    opts.add_argument("--disable-extensions")
+    opts.add_argument("--no-first-run")
+    opts.add_argument("--blink-settings=imagesEnabled=false")
+
+    if not chromium_path:
+        import shutil
+        for candidate in ("chromium", "chromium-browser", "google-chrome"):
+            found = shutil.which(candidate)
+            if found:
+                chromium_path = found
+                break
+    if chromium_path:
+        opts.binary_location = chromium_path
+
+    if not chromedriver_path:
+        import shutil
+        found = shutil.which("chromedriver")
+        if found:
+            chromedriver_path = found
+
+    svc    = Service(chromedriver_path) if chromedriver_path else Service()
+    driver = webdriver.Chrome(service=svc, options=opts)
+    driver.set_page_load_timeout(15)
+    return driver
+
+
+def _normalise(url: str) -> str:
+    """Strip fragment for deduplication."""
+    parsed = up.urlparse(url)
+    return up.urlunparse(parsed._replace(fragment="")).rstrip("/")