unseen 0.1.0__py3-none-any.whl

unseen/api.py

@@ -0,0 +1,133 @@
+"""HTTP API: the same scan as the CLI, over POST /scan."""
+
+from __future__ import annotations
+
+import ipaddress
+import math
+import os
+import threading
+from typing import Literal
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field, model_validator
+
+from unseen.cli import overall, scan_page
+from unseen.detectors.base import Page, Severity
+from unseen.fetcher import FetchError, fetch
+from unseen.net_guard import make_validator
+from unseen.ratelimit import RateLimiter
+from unseen.render import RenderError, render_hidden_text
+from unseen.sanitize import sanitize as sanitize_page
+
+MAX_HTML_CHARS = 2_000_000
+MAX_BODY_BYTES = 3_000_000
+
+app = FastAPI(title="unseen", version="0.1.0")
+RENDER_LOCK = threading.Lock()  # only one browser render at a time
+URL_VALIDATOR = make_validator()  # SSRF guard: runs on the first URL and on every redirect
+
+SCAN_LIMITER = RateLimiter(limit=10, window=60)  # scans per visitor per minute
+RENDER_LIMITER = RateLimiter(limit=3, window=60)  # browser renders per visitor per minute
+
+# Set UNSEEN_TRUST_PROXY=1 ONLY when a reverse proxy (Caddy) sits in front of the API.
+# Otherwise visitors could fake X-Forwarded-For and dodge the rate limit.
+TRUST_PROXY = os.environ.get("UNSEEN_TRUST_PROXY") == "1"
+
+
+@app.middleware("http")
+async def limit_body_size(request: Request, call_next):
+    length = request.headers.get("content-length", "")
+    if length.isdigit() and int(length) > MAX_BODY_BYTES:
+        return JSONResponse({"detail": "request body too large"}, status_code=413)
+    return await call_next(request)
+
+
+def client_key(http_request: Request) -> str:
+    """The identity used for rate limiting: the visitor's IP (IPv6 grouped by /64)."""
+    host = http_request.client.host if http_request.client else "unknown"
+    if TRUST_PROXY:
+        forwarded = http_request.headers.get("x-forwarded-for", "")
+        if forwarded:
+            host = forwarded.split(",")[-1].strip() or host  # the entry our proxy added
+    try:
+        ip = ipaddress.ip_address(host)
+    except ValueError:
+        return host
+    if isinstance(ip, ipaddress.IPv6Address):
+        return str(ipaddress.ip_network(f"{ip}/64", strict=False))
+    return str(ip)
+
+
+class ScanRequest(BaseModel):
+    url: str | None = None
+    html: str | None = Field(default=None, max_length=MAX_HTML_CHARS)
+    render: bool = False
+    sanitize: bool = False
+    min_severity: Literal["low", "medium", "high"] = "low"
+
+    @model_validator(mode="after")
+    def check_source(self) -> ScanRequest:
+        if (self.url is None) == (self.html is None):
+            raise ValueError("provide exactly one of 'url' or 'html'")
+        if self.url is not None and not self.url.startswith(("http://", "https://")):
+            raise ValueError("url must start with http:// or https://")
+        return self
+
+
+@app.get("/health")
+def health() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@app.post("/scan")
+def scan(request: ScanRequest, http_request: Request) -> dict:
+    key = client_key(http_request)
+    wait = SCAN_LIMITER.check(key)
+    if wait == 0 and request.render:
+        wait = RENDER_LIMITER.check(key)
+    if wait > 0:
+        raise HTTPException(
+            status_code=429,
+            detail="too many requests, try again later",
+            headers={"Retry-After": str(math.ceil(wait))},
+        )
+
+    if request.render and not RENDER_LOCK.acquire(blocking=False):
+        raise HTTPException(status_code=429, detail="another render is running, try again soon")
+
+    try:
+        if request.url is not None:
+            page = fetch(request.url, validator=URL_VALIDATOR)
+        else:
+            page = Page.from_html(request.html or "")
+        if request.render:
+            page.hidden_text = render_hidden_text(page.html)
+    except FetchError as error:
+        raise HTTPException(status_code=502, detail=f"could not fetch url: {error}") from error
+    except RenderError as error:
+        raise HTTPException(status_code=503, detail=f"rendering unavailable: {error}") from error
+    finally:
+        if request.render:
+            RENDER_LOCK.release()
+
+    merged = scan_page(page)
+    top = overall(merged)
+    minimum = Severity[request.min_severity.upper()]
+
+    result: dict = {
+        "target": request.url if request.url is not None else "(html)",
+        "overall": top.name.lower() if top else None,
+        "findings": [m.to_dict() for m in merged if m.severity >= minimum],
+    }
+
+    if request.sanitize:
+        cleaned = sanitize_page(page)
+        result["sanitized"] = {
+            "text": cleaned.text,
+            "removed": cleaned.removed,
+            "comments_removed": cleaned.comments_removed,
+            "invisible_removed": cleaned.invisible_removed,
+        }
+
+    return result

unseen/cli.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import unseen.detectors  # noqa: F401  (registers all detectors)
+from unseen.detectors.base import Page, Severity, all_detectors
+from unseen.fetcher import FetchError, fetch
+from unseen.render import RenderError, render_hidden_text
+from unseen.report import render as print_report
+from unseen.sanitize import sanitize
+from unseen.scoring import MergedFinding, merge_findings
+
+EXIT_OK = 0
+EXIT_RISK = 1
+EXIT_ERROR = 2
+
+
+def load_page(target: str) -> Page:
+    """Return a Page for a URL (via fetcher.py) or a local file path."""
+    if target.startswith(("http://", "https://")):
+        return fetch(target)
+    path = Path(target)
+    html = path.read_text(encoding="utf-8", errors="replace")
+    return Page.from_html(html, url=str(path))
+
+
+def scan_page(page: Page) -> list[MergedFinding]:
+    findings = []
+    for detector in all_detectors():
+        findings.extend(detector.scan(page))
+    return merge_findings(findings)
+
+
+def scan_target(target: str) -> list[MergedFinding]:
+    return scan_page(load_page(target))
+
+
+def overall(merged: list[MergedFinding]) -> Severity | None:
+    return max((m.severity for m in merged), default=None)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="unseen",
+        description="Find hidden text on a web page that humans cannot see but AI models can.",
+    )
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    scan = commands.add_parser("scan", help="scan a URL or an HTML file")
+    scan.add_argument("target", help="URL or path to an HTML file")
+    scan.add_argument("--json", action="store_true", help="print results as JSON")
+    scan.add_argument(
+        "--min-severity",
+        choices=["low", "medium", "high"],
+        default="low",
+        help="only show findings at or above this level",
+    )
+    scan.add_argument(
+        "--fail-on",
+        choices=["low", "medium", "high"],
+        default="high",
+        help="exit with code 1 if any finding is at or above this level (default: high)",
+    )
+    scan.add_argument(
+        "--sanitize",
+        metavar="FILE",
+        help="write a cleaned text version of the page (hidden content removed) to FILE",
+    )
+    scan.add_argument(
+        "--render",
+        action="store_true",
+        help="also render the page in a headless browser to catch hidden text that "
+        "static analysis misses (needs Playwright)",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    try:
+        page = load_page(args.target)
+        if args.render:
+            page.hidden_text = render_hidden_text(page.html)
+        merged = scan_page(page)
+        cleaned = sanitize(page) if args.sanitize else None
+    except FetchError as error:
+        print(f"unseen: could not fetch {args.target}: {error}", file=sys.stderr)
+        return EXIT_ERROR
+    except RenderError as error:
+        print(f"unseen: {error}", file=sys.stderr)
+        return EXIT_ERROR
+    except Exception as error:  # file or decoding problems
+        print(f"unseen: could not scan {args.target}: {error}", file=sys.stderr)
+        return EXIT_ERROR
+
+    fail_level = Severity[args.fail_on.upper()]
+    top = overall(merged)
+    failed = top is not None and top >= fail_level
+
+    shown = [m for m in merged if m.severity >= Severity[args.min_severity.upper()]]
+
+    if args.json:
+        output = {
+            "target": args.target,
+            "overall": top.name.lower() if top else None,
+            "findings": [m.to_dict() for m in shown],
+        }
+        print(json.dumps(output, indent=2))
+    else:
+        print_report(args.target, shown)
+
+    if cleaned is not None:
+        try:
+            Path(args.sanitize).write_text(cleaned.text + "\n", encoding="utf-8")
+        except OSError as error:
+            print(f"unseen: could not write {args.sanitize}: {error}", file=sys.stderr)
+            return EXIT_ERROR
+        print(
+            f"unseen: cleaned text written to {args.sanitize} "
+            f"({len(cleaned.removed)} hidden elements, {cleaned.comments_removed} comments, "
+            f"{cleaned.invisible_removed} invisible characters removed)",
+            file=sys.stderr,
+        )
+
+    return EXIT_RISK if failed else EXIT_OK
+
+
+if __name__ == "__main__":
+    sys.exit(main())

unseen/detectors/__init__.py

@@ -0,0 +1,8 @@
+from unseen.detectors import (  # noqa: F401
+    css_hiding,
+    hidden_attrs,
+    html_comments,
+    instruction_patterns,
+    invisible_unicode,
+    render_diff,
+)

unseen/detectors/base.py

@@ -0,0 +1,92 @@
+"""Shared types and the base class every detector implements."""
+
+from __future__ import annotations
+
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import TYPE_CHECKING
+
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+if TYPE_CHECKING:
+    from unseen.render import HiddenText
+
+
+class Severity(IntEnum):
+    LOW = 1
+    MEDIUM = 2
+    HIGH = 3
+
+
+def clip(text: str, limit: int = 120) -> str:
+    """Collapse whitespace and shorten text for display."""
+    text = re.sub(r"\s+", " ", text).strip()
+    return text if len(text) <= limit else text[: limit - 3] + "..."
+
+
+def describe(tag: Tag) -> str:
+    """Short CSS-like label for an element, e.g. div#note.footer."""
+    label = tag.name
+    if tag.get("id"):
+        label += f"#{tag['id']}"
+    classes = tag.get("class")
+    if classes:
+        label += "." + ".".join(classes)
+    return label
+
+
+@dataclass(frozen=True)
+class Finding:
+    detector: str
+    severity: Severity
+    message: str
+    location: str
+    snippet: str = ""
+
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "detector": self.detector,
+            "severity": self.severity.name.lower(),
+            "message": self.message,
+            "location": self.location,
+            "snippet": self.snippet,
+        }
+
+
+@dataclass
+class Page:
+    url: str
+    html: str
+    soup: BeautifulSoup
+    hidden_text: list[HiddenText] | None = None  # filled by render.py when --render is used
+
+    @classmethod
+    def from_html(cls, html: str, url: str = "") -> Page:
+        return cls(url=url, html=html, soup=BeautifulSoup(html, "lxml"))
+
+
+class Detector(ABC):
+    """One hiding technique. Subclasses set name and description, then implement scan."""
+
+    name: str
+    description: str
+
+    @abstractmethod
+    def scan(self, page: Page) -> list[Finding]:
+        """Return every finding for this technique on the page."""
+
+
+REGISTRY: dict[str, type[Detector]] = {}
+
+
+def register(cls: type[Detector]) -> type[Detector]:
+    """Class decorator that adds a detector to the registry."""
+    REGISTRY[cls.name] = cls
+    return cls
+
+
+def all_detectors() -> list[Detector]:
+    return [cls() for cls in REGISTRY.values()]

unseen/detectors/css_hiding.py

@@ -0,0 +1,226 @@
+"""Detector 1: text hidden from humans with CSS or the hidden attribute."""
+
+from __future__ import annotations
+
+import re
+
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+from .base import Detector, Finding, Page, Severity, clip, describe, register
+from .instruction_patterns import detect_instruction_patterns
+
+SKIP_TAGS = {"script", "style", "noscript", "template", "head", "title", "meta", "link"}
+
+# Common on legitimate pages (menus, modals), so they score lower than deceptive tricks.
+ORDINARY = {"hidden attribute", "display:none", "visibility:hidden"}
+
+# Screen-reader-only text uses these. With very short text they cannot carry an attack.
+SCREEN_READER = {"clipped to nothing", "zero-size container"}
+
+# A closed dropdown or accordion: hidden with display/visibility AND collapsed to zero size.
+COLLAPSE_HIDERS = {"display:none", "visibility:hidden", "hidden attribute"}
+
+NAMED_COLORS = {
+    "white": (255, 255, 255),
+    "black": (0, 0, 0),
+    "red": (255, 0, 0),
+    "green": (0, 128, 0),
+    "blue": (0, 0, 255),
+    "yellow": (255, 255, 0),
+    "gray": (128, 128, 128),
+    "grey": (128, 128, 128),
+    "silver": (192, 192, 192),
+}
+
+COLOR_RE = re.compile(
+    r"#[0-9a-f]{6}\b|#[0-9a-f]{3}\b|rgba?\([^)]*\)|\b(?:" + "|".join(NAMED_COLORS) + r")\b"
+)
+RULE_RE = re.compile(r"([^{}@]+)\{([^{}]*)\}")
+CLIP_RECT_RE = re.compile(r"rect\(\s*0(?:px)?[\s,]+0(?:px)?[\s,]+0(?:px)?[\s,]+0(?:px)?\s*\)")
+TRANSPARENT_RE = re.compile(r"rgba\(\d+,\d+,\d+,0(?:\.0+)?\)")
+
+
+def number(value: str) -> float | None:
+    match = re.match(r"\s*(-?\d*\.?\d+)", value)
+    return float(match.group(1)) if match else None
+
+
+def parse_color(value: str) -> tuple[int, ...] | None:
+    value = value.strip().lower().replace(" ", "")
+    if value in NAMED_COLORS:
+        return NAMED_COLORS[value]
+    match = re.fullmatch(r"#([0-9a-f]{3}|[0-9a-f]{6})", value)
+    if match:
+        digits = match.group(1)
+        if len(digits) == 3:
+            digits = "".join(c * 2 for c in digits)
+        return tuple(int(digits[i : i + 2], 16) for i in (0, 2, 4))
+    match = re.fullmatch(r"rgba?\((\d+),(\d+),(\d+)(?:,[\d.]+)?\)", value)
+    if match:
+        return tuple(int(g) for g in match.groups())
+    return None
+
+
+def find_color(value: str) -> tuple[int, ...] | None:
+    match = COLOR_RE.search(value.lower())
+    return parse_color(match.group(0)) if match else None
+
+
+def is_transparent(value: str) -> bool:
+    value = value.replace(" ", "")
+    return value == "transparent" or bool(TRANSPARENT_RE.fullmatch(value))
+
+
+def parse_declarations(block: str) -> dict[str, str]:
+    declarations: dict[str, str] = {}
+    for part in block.split(";"):
+        if ":" not in part:
+            continue
+        prop, _, value = part.partition(":")
+        value = re.sub(r"\s*!important\s*$", "", value.strip(), flags=re.I)
+        declarations[prop.strip().lower()] = value.lower()
+    return declarations
+
+
+def compute_styles(soup: BeautifulSoup) -> dict[int, dict[str, str]]:
+    """Map id(tag) to declarations from <style> blocks. Specificity is ignored (last rule wins)."""
+    css = "\n".join(style.get_text() for style in soup.find_all("style"))
+    css = re.sub(r"/\*.*?\*/", "", css, flags=re.S)
+    styles: dict[int, dict[str, str]] = {}
+    for selector, body in RULE_RE.findall(css):
+        try:
+            matches = soup.select(selector.strip())
+        except Exception:
+            continue
+        declarations = parse_declarations(body)
+        for tag in matches:
+            styles.setdefault(id(tag), {}).update(declarations)
+    return styles
+
+
+def hiding_reasons(tag: Tag, style: dict[str, str]) -> list[str]:
+    reasons: list[str] = []
+
+    if tag.has_attr("hidden"):
+        reasons.append("hidden attribute")
+    if style.get("display") == "none":
+        reasons.append("display:none")
+    if style.get("visibility") in ("hidden", "collapse"):
+        reasons.append("visibility:hidden")
+
+    opacity = number(style.get("opacity", ""))
+    if opacity is not None and opacity <= 0.01:
+        reasons.append("opacity:0")
+
+    raw_size = style.get("font-size", "")
+    font_size = number(raw_size)
+    if font_size is not None and (font_size == 0 or (font_size <= 1 and "px" in raw_size)):
+        reasons.append("font-size:0")
+
+    indent = number(style.get("text-indent", ""))
+    if indent is not None and indent <= -500:
+        reasons.append("text-indent off-screen")
+
+    if style.get("position") in ("absolute", "fixed", "relative"):
+        for side in ("left", "top", "right", "bottom"):
+            offset = number(style.get(side, ""))
+            if offset is not None and offset <= -500:
+                reasons.append("off-screen position")
+                break
+
+    clip_path = style.get("clip-path", "").replace(" ", "")
+    if CLIP_RECT_RE.fullmatch(style.get("clip", "").strip()):
+        reasons.append("clipped to nothing")
+    elif clip_path.startswith(("inset(100%", "inset(50%")):
+        reasons.append("clipped to nothing")
+
+    if style.get("overflow") == "hidden":
+        width = number(style.get("width", ""))
+        height = number(style.get("height", ""))
+        if (width is not None and width <= 1) or (height is not None and height <= 1):
+            reasons.append("zero-size container")
+
+    color = style.get("color", "")
+    if color and is_transparent(color):
+        reasons.append("transparent text")
+    else:
+        fg = find_color(color) if color else None
+        bg_value = style.get("background-color") or style.get("background") or ""
+        bg = find_color(bg_value) if bg_value else None
+        if fg is not None and fg == bg:
+            reasons.append("text color equals background")
+
+    return reasons
+
+
+def is_skip_link(tag: Tag, text: str) -> bool:
+    """A short in-page anchor such as 'Skip to content' (href starting with #)."""
+    return tag.name == "a" and str(tag.get("href", "")).startswith("#") and len(text) < 40
+
+
+def is_ordinary_reason(reason: str, reasons: list[str], tag: Tag, text: str) -> bool:
+    if reason in ORDINARY:
+        return True
+    short = len(text) < 60
+    if short and reason in SCREEN_READER:
+        return True
+    # A closed dropdown or accordion: collapsed to zero size AND hidden the usual way.
+    if reason == "zero-size container" and any(r in COLLAPSE_HIDERS for r in reasons):
+        return True
+    # Skip links are moved off-screen until they receive focus.
+    if reason == "off-screen position" and is_skip_link(tag, text):
+        return True
+    return False
+
+
+@register
+class CssHiding(Detector):
+    name = "css_hiding"
+    description = "Text hidden from humans with CSS or the hidden attribute"
+
+    def scan(self, page: Page) -> list[Finding]:
+        styles = compute_styles(page.soup)
+        findings: list[Finding] = []
+        flagged: set[int] = set()
+
+        for tag in page.soup.find_all(True):
+            if tag.name in SKIP_TAGS:
+                continue
+            if any(id(parent) in flagged for parent in tag.parents):
+                continue
+
+            style = dict(styles.get(id(tag), {}))
+            style.update(parse_declarations(tag.get("style", "") or ""))
+
+            reasons = hiding_reasons(tag, style)
+            if not reasons:
+                continue
+
+            text = tag.get_text(" ", strip=True)
+            if not text:
+                continue
+
+            flagged.add(id(tag))
+            deceptive = [r for r in reasons if not is_ordinary_reason(r, reasons, tag, text)]
+            if deceptive:
+                severity = Severity.HIGH if len(text) >= 20 else Severity.MEDIUM
+            else:
+                severity = Severity.LOW
+
+            message = f"Hidden from humans via {', '.join(reasons)}"
+            # An instruction anywhere inside hidden content (children included) is HIGH.
+            if detect_instruction_patterns(text):
+                severity = Severity.HIGH
+                message += " (contains instruction-like text)"
+
+            findings.append(
+                Finding(
+                    detector=self.name,
+                    severity=severity,
+                    message=message,
+                    location=describe(tag),
+                    snippet=clip(text),
+                )
+            )
+        return findings

unseen/detectors/hidden_attrs.py

@@ -0,0 +1,74 @@
+"""Detector 4: text aimed at machines and assistive readers rather than at human eyes."""
+
+from __future__ import annotations
+
+from .base import Detector, Finding, Page, Severity, clip, describe, register
+from .patterns import find_instruction
+
+SKIP_ATTRS = {"href", "src", "srcset", "action", "style", "class", "id", "rel", "type", "name"}
+LONG_ATTRS = {"alt", "title", "aria-label", "aria-description"}
+MIN_LENGTH = 8
+LONG_TEXT = 200
+
+
+def attr_text(value: object) -> str:
+    return " ".join(value) if isinstance(value, list) else str(value)
+
+
+@register
+class HiddenAttrs(Detector):
+    name = "hidden_attrs"
+    description = "Instructions in attributes, meta tags, noscript and aria-hidden content"
+
+    def scan(self, page: Page) -> list[Finding]:
+        findings: list[Finding] = []
+        flagged: set[int] = set()
+
+        def add(severity: Severity, message: str, location: str, snippet: str) -> None:
+            findings.append(Finding(self.name, severity, message, location, snippet))
+
+        for tag in page.soup.find_all(True):
+            for attr, value in tag.attrs.items():
+                if attr in SKIP_ATTRS:
+                    continue
+                text = attr_text(value)
+                if len(text) < MIN_LENGTH:
+                    continue
+                location = f"{describe(tag)}[{attr}]"
+                phrase = find_instruction(text)
+                if phrase:
+                    message = (
+                        f'Attribute "{attr}" contains instruction-like text: '
+                        f'"{clip(phrase, 60)}"'
+                    )
+                    add(Severity.HIGH, message, location, clip(text))
+                elif attr in LONG_ATTRS and len(text) >= LONG_TEXT:
+                    message = f'Unusually long "{attr}" text ({len(text)} characters)'
+                    add(Severity.LOW, message, location, clip(text))
+
+            if any(id(parent) in flagged for parent in tag.parents):
+                continue
+
+            if str(tag.get("aria-hidden", "")).lower() == "true":
+                text = tag.get_text(" ", strip=True)
+                phrase = find_instruction(text)
+                if phrase:
+                    flagged.add(id(tag))
+                    message = (
+                        "aria-hidden content contains instruction-like text: "
+                        f'"{clip(phrase, 60)}"'
+                    )
+                    add(Severity.MEDIUM, message, describe(tag), clip(text))
+
+            if tag.name == "noscript":
+                text = tag.get_text(" ", strip=True)
+                phrase = find_instruction(text)
+                if phrase:
+                    flagged.add(id(tag))
+                    message = (
+                        "noscript content contains instruction-like text: "
+                        f'"{clip(phrase, 60)}"'
+                    )
+                    add(Severity.HIGH, message, describe(tag), clip(text))
+
+        return findings