theus-cli-linux-x64 1.2.1 → 1.3.0

package/vendor/higiene-motor/audit_website.py

@@ -0,0 +1,635 @@
+#!/usr/bin/env python3
+"""Aggregate AI-provenance audit over the URLs listed in a sitemap.
+
+Stdlib-only: downloads each URL, classifies it by content type/suffix/magic,
+and runs the same deterministic text/image/container inspections used by the
+local audit. Optional external tools (c2patool/exiftool) are not invoked for
+remote URLs; download the assets and run audit_dir.py locally for those.
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import http.client
+import io
+import ipaddress
+import socket
+import ssl
+import sys
+import tempfile
+import urllib.error
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import contextlib
+
+from audit_lib import aggregate, print_human_report, scan_file
+from common import EXIT_PARTIAL, emit_json, eprint
+
+DEFAULT_MAX_BYTES = 4 << 20
+DEFAULT_TIMEOUT = 15
+DEFAULT_MAX_PAGES = 200
+MAX_SITEMAP_DECOMPRESSED_BYTES = 64 << 20
+MAX_REDIRECTS = 5
+USER_AGENT = "remove-ai-marks-audit/1.0"
+
+_EXT_FOR_KIND = {
+    "webp": ".webp",
+    "avif": ".avif",
+    "heic": ".heic",
+    "gif": ".gif",
+    "bmp": ".bmp",
+    "tiff": ".tiff",
+    "xlsx": ".xlsx",
+    "pptx": ".pptx",
+    "epub": ".epub",
+    "mp4": ".mp4",
+    "wav": ".wav",
+    "mp3": ".mp3",
+    "png": ".png",
+    "jpeg": ".jpg",
+    "svg": ".svg",
+    "pdf": ".pdf",
+    "docx": ".docx",
+    "odt": ".odt",
+    "html": ".html",
+    "markdown": ".md",
+    "text": ".txt",
+}
+
+
+def _local(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1]
+
+
+def parse_sitemap(data: bytes) -> tuple[str, list[str]]:
+    """Parse a (possibly gzip-compressed) sitemap into (kind, urls)."""
+    if data[:2] == b"\x1f\x8b":
+        with gzip.GzipFile(fileobj=io.BytesIO(data)) as stream:
+            data = stream.read(MAX_SITEMAP_DECOMPRESSED_BYTES + 1)
+        if len(data) > MAX_SITEMAP_DECOMPRESSED_BYTES:
+            raise ValueError(
+                f"sitemap decompressed size exceeds cap ({MAX_SITEMAP_DECOMPRESSED_BYTES} bytes)"
+            )
+    elif len(data) > MAX_SITEMAP_DECOMPRESSED_BYTES:
+        raise ValueError(f"sitemap size exceeds cap ({MAX_SITEMAP_DECOMPRESSED_BYTES} bytes)")
+
+    # This repo is stdlib-only by policy, so sitemaps use ElementTree. stdlib
+    # ET does not expand external entities (no XXE), but internal entity
+    # expansion (billion-laughs) is still possible, so DTDs/entities are
+    # rejected outright before parsing. If the policy ever changes, swap in
+    # defusedxml.ElementTree with the same DTD rejection as defense in depth.
+    if b"<!DOCTYPE" in data or b"<!ENTITY" in data:
+        raise ValueError("sitemap declares a DTD / entities; refusing to parse")
+
+    root = ET.fromstring(data)  # noqa: S314 - DTD/entity declarations rejected above
+    kind = _local(root.tag)
+    urls = []
+    for el in root.iter():
+        if _local(el.tag) == "loc" and el.text:
+            urls.append(el.text.strip())
+    return kind, urls
+
+
+def guess_kind(url: str, data: bytes, content_type: str | None = None) -> str:
+    """Classify a downloaded URL from headers, suffix, then magic bytes."""
+    ct = (content_type or "").lower().split(";")[0].strip()
+    if "html" in ct:
+        return "html"
+    if ct == "image/png":
+        return "png"
+    if ct == "image/jpeg":
+        return "jpeg"
+    if "svg" in ct:
+        return "svg"
+    if ct == "application/pdf":
+        return "pdf"
+    if "wordprocessingml" in ct:
+        return "docx"
+    if "opendocument.text" in ct:
+        return "odt"
+    if "markdown" in ct:
+        return "markdown"
+    if ct == "text/plain":
+        return "text"
+    # The formats the local audit (format_dispatch) handles arrived over HTTP
+    # as "text" before this table knew them: the bytes fell to the Unicode
+    # scanner and binary assets reported clean (#166).
+    if ct in (
+        "image/webp",
+        "image/avif",
+        "image/heic",
+        "image/heif",
+        "image/gif",
+        "image/bmp",
+        "image/tiff",
+    ):
+        return {
+            "image/webp": "webp",
+            "image/avif": "avif",
+            "image/heic": "heic",
+            "image/heif": "heic",
+            "image/gif": "gif",
+            "image/bmp": "bmp",
+            "image/tiff": "tiff",
+        }[ct]
+    if ct == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+        return "xlsx"
+    if ct == "application/vnd.openxmlformats-officedocument.presentationml.presentation":
+        return "pptx"
+    if ct == "application/epub+zip":
+        return "epub"
+    if ct in ("video/mp4", "video/quicktime", "audio/x-m4a", "audio/mp4"):
+        return "mp4"
+    if ct in ("audio/wav", "audio/x-wav", "audio/mpeg", "audio/mp3"):
+        return {"audio/wav": "wav", "audio/x-wav": "wav", "audio/mpeg": "mp3", "audio/mp3": "mp3"}[
+            ct
+        ]
+
+    path = urllib.parse.urlparse(url).path.lower()
+    for ext, kind in (
+        (".png", "png"),
+        (".jpg", "jpeg"),
+        (".jpeg", "jpeg"),
+        (".webp", "webp"),
+        (".avif", "avif"),
+        (".heic", "heic"),
+        (".heif", "heic"),
+        (".gif", "gif"),
+        (".bmp", "bmp"),
+        (".tiff", "tiff"),
+        (".tif", "tiff"),
+        (".svg", "svg"),
+        (".pdf", "pdf"),
+        (".docx", "docx"),
+        (".xlsx", "xlsx"),
+        (".pptx", "pptx"),
+        (".odt", "odt"),
+        (".epub", "epub"),
+        (".mp4", "mp4"),
+        (".mov", "mp4"),
+        (".m4a", "mp4"),
+        (".wav", "wav"),
+        (".mp3", "mp3"),
+        (".html", "html"),
+        (".htm", "html"),
+        (".md", "markdown"),
+        (".markdown", "markdown"),
+        (".txt", "text"),
+    ):
+        if path.endswith(ext):
+            return kind
+
+    if data.startswith(b"\x89PNG"):
+        return "png"
+    if data.startswith(b"\xff\xd8"):
+        return "jpeg"
+    if data.startswith(b"%PDF"):
+        return "pdf"
+    if data.startswith(b"RIFF") and data[8:12] == b"WEBP":
+        return "webp"
+    if data.startswith(b"GIF8"):
+        return "gif"
+    if data.startswith(b"BM"):
+        return "bmp"
+    if data.startswith(b"II*\x00") or data.startswith(b"MM\x00*"):
+        return "tiff"
+    if data[4:8] == b"ftyp":
+        brand = data[8:12]
+        if brand in (b"avif", b"avis"):
+            return "avif"
+        if brand in (b"heic", b"heix", b"hevc", b"hevx", b"mif1", b"msf1"):
+            return "heic"
+        return "mp4"
+    if data[:100].lstrip().startswith(b"<") and b"svg" in data[:500].lower():
+        return "svg"
+    if b"<html" in data[:2000].lower() or data[:100].lstrip().lower().startswith(b"<"):
+        return "html"
+    return "text"
+
+
+UrlOrigin = tuple[str, str, int]
+
+
+def _url_origin(url: str) -> UrlOrigin:
+    """Return a normalized HTTP(S) origin or reject unsafe URL forms."""
+    parsed = urllib.parse.urlsplit(url)
+    scheme = parsed.scheme.lower()
+
+    if scheme not in ("http", "https"):
+        raise ValueError(f"unsupported URL scheme: {scheme or '(missing)'}")
+
+    if parsed.username is not None or parsed.password is not None:
+        raise ValueError("credentials in URLs are not allowed")
+
+    host = parsed.hostname
+    if not host:
+        raise ValueError("URL has no hostname")
+
+    try:
+        port = parsed.port
+    except ValueError as exc:
+        raise ValueError(f"invalid URL port: {exc}") from exc
+
+    if port is None:
+        port = 443 if scheme == "https" else 80
+
+    return scheme, host.rstrip(".").lower(), port
+
+
+def _origin_allowed(candidate: UrlOrigin, expected: UrlOrigin) -> bool:
+    """Allow same-origin URLs plus a normal HTTP-to-HTTPS upgrade."""
+    if candidate == expected:
+        return True
+    return (
+        expected[0] == "http"
+        and expected[2] == 80
+        and candidate[0] == "https"
+        and candidate[2] == 443
+        and candidate[1] == expected[1]
+    )
+
+
+def _resolve_public_addresses(origin: UrlOrigin) -> tuple[str, ...]:
+    """Resolve an origin to validated public numeric addresses."""
+    _scheme, host, port = origin
+    host_for_ip = host.split("%", 1)[0]
+    raw_addresses: list[str] = []
+
+    try:
+        raw_addresses.append(str(ipaddress.ip_address(host_for_ip)))
+    except ValueError:
+        try:
+            infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
+        except socket.gaierror as exc:
+            raise ValueError(f"cannot resolve hostname {host}: {exc}") from exc
+
+        for info in infos:
+            raw_addresses.append(str(info[4][0]).split("%", 1)[0])
+
+    addresses: list[str] = []
+    for raw in raw_addresses:
+        try:
+            address = ipaddress.ip_address(raw)
+        except ValueError:
+            continue
+
+        mapped = getattr(address, "ipv4_mapped", None)
+        if mapped is not None:
+            address = mapped
+
+        if not address.is_global:
+            raise ValueError(f"refusing non-public address for {host}: {address}")
+
+        canonical = str(address)
+        if canonical not in addresses:
+            addresses.append(canonical)
+
+    if not addresses:
+        raise ValueError(f"hostname resolved to no IP addresses: {host}")
+
+    return tuple(addresses)
+
+
+def _validated_target(
+    url: str,
+    *,
+    expected_origin: UrlOrigin | None = None,
+) -> tuple[UrlOrigin, tuple[str, ...]]:
+    """Validate URL policy and bind it to numeric public IPs."""
+    origin = _url_origin(url)
+
+    if expected_origin is not None and not _origin_allowed(origin, expected_origin):
+        raise ValueError(f"cross-origin URL is not allowed: {origin[0]}://{origin[1]}:{origin[2]}")
+
+    return origin, _resolve_public_addresses(origin)
+
+
+def _validate_public_http_url(
+    url: str,
+    *,
+    expected_origin: UrlOrigin | None = None,
+) -> UrlOrigin:
+    """Validate a public HTTP(S) URL and return its origin."""
+    origin, _addresses = _validated_target(url, expected_origin=expected_origin)
+    return origin
+
+
+def _open_pinned_connection(
+    origin: UrlOrigin,
+    addresses: tuple[str, ...],
+    timeout: int,
+) -> http.client.HTTPConnection:
+    """Connect to a validated IP while preserving Host and TLS SNI."""
+    scheme, host, port = origin
+    last_error: OSError | None = None
+
+    for address in addresses:
+        raw = None
+        try:
+            raw = socket.create_connection((address, port), timeout=timeout)
+
+            if scheme == "https":
+                context = ssl.create_default_context()
+                context.minimum_version = ssl.TLSVersion.TLSv1_2
+                with contextlib.suppress(NotImplementedError):
+                    context.set_alpn_protocols(["http/1.1"])
+
+                wrapped = context.wrap_socket(raw, server_hostname=host)
+                raw = None
+
+                conn = http.client.HTTPSConnection(
+                    host,
+                    port,
+                    timeout=timeout,
+                    context=context,
+                )
+                conn.sock = wrapped
+                return conn
+
+            conn = http.client.HTTPConnection(host, port, timeout=timeout)
+            conn.sock = raw
+            raw = None
+            return conn
+
+        except OSError as exc:
+            last_error = exc
+            if raw is not None:
+                with contextlib.suppress(OSError):
+                    raw.close()
+
+    if last_error is not None:
+        raise last_error
+    raise OSError(f"no validated address available for {host}")
+
+
+def _request_target(url: str) -> str:
+    parsed = urllib.parse.urlsplit(url)
+    target = parsed.path or "/"
+    if parsed.query:
+        target += "?" + parsed.query
+    return target
+
+
+def fetch(
+    url: str,
+    timeout: int,
+    max_bytes: int,
+    *,
+    allowed_origin: UrlOrigin | None = None,
+) -> tuple[bytes, str | None]:
+    """Fetch with IP pinning, redirect validation and a byte cap."""
+    current_url = url
+    expected_origin = allowed_origin
+
+    for redirect_count in range(MAX_REDIRECTS + 1):
+        origin, addresses = _validated_target(
+            current_url,
+            expected_origin=expected_origin,
+        )
+
+        if expected_origin is None:
+            expected_origin = origin
+
+        conn = _open_pinned_connection(origin, addresses, timeout)
+        response = None
+
+        try:
+            conn.request(
+                "GET",
+                _request_target(current_url),
+                headers={
+                    "User-Agent": USER_AGENT,
+                    "Connection": "close",
+                },
+            )
+            response = conn.getresponse()
+
+            if response.status in (301, 302, 303, 307, 308):
+                location = response.getheader("Location")
+                if location:
+                    if redirect_count >= MAX_REDIRECTS:
+                        raise ValueError(f"too many redirects (>{MAX_REDIRECTS})")
+                    current_url = urllib.parse.urljoin(current_url, location)
+                    continue
+
+            content_type = response.getheader("Content-Type")
+            chunks = []
+            total = 0
+
+            while True:
+                chunk = response.read(1 << 16)
+                if not chunk:
+                    break
+                total += len(chunk)
+                if total > max_bytes:
+                    raise ValueError(f"exceeds {max_bytes} bytes")
+                chunks.append(chunk)
+
+            return b"".join(chunks), content_type
+
+        finally:
+            if response is not None:
+                response.close()
+            conn.close()
+
+    raise ValueError(f"too many redirects (>{MAX_REDIRECTS})")
+
+
+def inspect_remote(url: str, data: bytes, content_type: str | None = None) -> dict:
+    """Inspect downloaded bytes using the local scan_file pipeline."""
+    kind = guess_kind(url, data, content_type)
+    ext = _EXT_FOR_KIND.get(kind, ".bin")
+    with tempfile.TemporaryDirectory() as td:
+        tmp = Path(td) / f"asset{ext}"
+        tmp.write_bytes(data)
+        result = scan_file(tmp, display_name=url)
+    result["kind"] = kind
+    return result
+
+
+def discover_sitemap(base_url: str, timeout: int) -> str | None:
+    """Find a same-site sitemap via standard paths then robots.txt."""
+    origin = _validate_public_http_url(base_url)
+    base = base_url.rstrip("/")
+
+    for candidate in (
+        f"{base}/sitemap.xml",
+        f"{base}/sitemap_index.xml",
+    ):
+        try:
+            data, _ = fetch(
+                candidate,
+                timeout,
+                DEFAULT_MAX_BYTES,
+                allowed_origin=origin,
+            )
+            parse_sitemap(data)
+            return candidate
+        except Exception:  # noqa: S112 - try next candidate sitemap on any failure
+            continue
+
+    try:
+        data, _ = fetch(
+            f"{base}/robots.txt",
+            timeout,
+            1 << 20,
+            allowed_origin=origin,
+        )
+
+        text = data.decode("utf-8", errors="replace")
+
+        for line in text.splitlines():
+            if line.lower().startswith("sitemap:"):
+                candidate = line.split(":", 1)[1].strip()
+                candidate_origin = _url_origin(candidate)
+
+                if not _origin_allowed(candidate_origin, origin):
+                    raise ValueError(f"cross-origin sitemap is not allowed: {candidate}")
+
+                return candidate
+
+    except Exception:  # noqa: S110 - robots.txt is optional; fall through to None
+        pass
+
+    return None
+
+
+def collect_urls(
+    sitemap_url: str,
+    timeout: int,
+    max_pages: int,
+) -> list[str]:
+    """Collect same-site URLs while following nested sitemap indexes."""
+    urls: list[str] = []
+    seen: set[str] = set()
+    origin = _validate_public_http_url(sitemap_url)
+
+    def _check_loc(loc: str) -> None:
+        candidate_origin = _url_origin(loc)
+
+        if not _origin_allowed(candidate_origin, origin):
+            raise ValueError(f"cross-origin sitemap URL is not allowed: {loc}")
+
+    def _recurse(url: str, depth: int = 0) -> None:
+        if len(urls) >= max_pages or depth > 3:
+            return
+
+        data, _ = fetch(
+            url,
+            timeout,
+            DEFAULT_MAX_BYTES,
+            allowed_origin=origin,
+        )
+
+        kind, locs = parse_sitemap(data)
+
+        if kind == "sitemapindex":
+            for loc in locs:
+                _check_loc(loc)
+
+                if loc not in seen:
+                    seen.add(loc)
+                    _recurse(loc, depth + 1)
+
+        else:
+            for loc in locs:
+                _check_loc(loc)
+
+                if loc not in seen:
+                    seen.add(loc)
+                    urls.append(loc)
+
+                    if len(urls) >= max_pages:
+                        break
+
+    _recurse(sitemap_url)
+    return urls
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--sitemap", help="Sitemap URL to audit")
+    p.add_argument("--base", help="Base URL; discover the sitemap automatically")
+    p.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES)
+    p.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT)
+    p.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES)
+    p.add_argument("--json", action="store_true")
+    args = p.parse_args()
+
+    if not args.sitemap and not args.base:
+        eprint("provide --sitemap URL or --base URL")
+        return 2
+
+    sitemap_url = args.sitemap
+    if not sitemap_url:
+        try:
+            sitemap_url = discover_sitemap(args.base, args.timeout)
+        except ValueError as e:
+            eprint(f"invalid base URL: {e}")
+            return 2
+        if not sitemap_url:
+            eprint(f"no sitemap found for {args.base}")
+            return 2
+
+    try:
+        urls = collect_urls(sitemap_url, args.timeout, args.max_pages)
+    except Exception as e:
+        eprint(f"could not collect URLs from {sitemap_url}: {e}")
+        return 2
+    if not urls:
+        eprint("no URLs collected from sitemap")
+        return 2
+
+    files = []
+    failures = []
+    for url in urls[: args.max_pages]:
+        try:
+            data, content_type = fetch(url, args.timeout, args.max_bytes)
+        except Exception as e:
+            failures.append({"url": url, "error": str(e)})
+            continue
+        try:
+            files.append(inspect_remote(url, data, content_type))
+        except Exception as e:
+            failures.append({"url": url, "error": f"inspect failed: {e}"})
+
+    summary = aggregate(files)
+    report = {
+        "sitemap": sitemap_url,
+        "base": args.base,
+        "urls_collected": len(urls),
+        "urls_scanned": len(files),
+        "urls_failed": failures,
+        "summary": summary,
+        "files": files,
+    }
+
+    if args.json:
+        emit_json(report)
+    else:
+        print_human_report(
+            files,
+            summary,
+            extra_header={
+                "Sitemap": sitemap_url,
+                "URLs collected": str(len(urls)),
+                "URLs scanned": str(len(files)),
+                "URLs failed": str(len(failures)),
+            },
+        )
+        for failure in failures:
+            print(f"  [error] {failure['url']}: {failure['error']}")
+
+    # A partial scan (one or more URLs failed to fetch/inspect) gets a
+    # distinct code: an incomplete audit outranks actionable findings.
+    return EXIT_PARTIAL if failures else (1 if summary["actionable_files"] else 0)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())