captcha-detector 1.0.0__py3-none-any.whl

captcha_detector/__init__.py

@@ -0,0 +1,55 @@
+"""captcha-detector: does this web page have a captcha?
+
+    from captcha_detector import detect_html, detect_response
+
+    r = detect_response(requests.get(url), url=url)
+    if r.captcha or r.challenge_page or r.blocked:
+        print(r.summary)
+
+Browser helpers (Playwright, Selenium) are in captcha_detector.browser and re-exported here.
+"""
+
+from .engine import (
+    SIGNATURES,
+    VERSION,
+    Detection,
+    Result,
+    analyze,
+    detect_html,
+    from_js,
+    merge_results,
+    snapshot_from_html,
+)
+from .response import detect_response, detect_response_async
+from .browser import (
+    BROWSER_SCRIPT,
+    detect_in_page,
+    detect_in_page_async,
+    detect_selenium,
+    expression,
+    wait_for_captcha,
+    wait_for_captcha_async,
+)
+
+__version__ = VERSION
+
+__all__ = [
+    "VERSION",
+    "SIGNATURES",
+    "Detection",
+    "Result",
+    "detect_html",
+    "detect_response",
+    "detect_response_async",
+    "detect_in_page",
+    "detect_in_page_async",
+    "wait_for_captcha",
+    "wait_for_captcha_async",
+    "detect_selenium",
+    "merge_results",
+    "analyze",
+    "snapshot_from_html",
+    "from_js",
+    "expression",
+    "BROWSER_SCRIPT",
+]

captcha_detector/__main__.py

@@ -0,0 +1,112 @@
+"""Command line: python -m captcha_detector <url | file.html | -> [options]  (also installed as `captcha-detector`)."""
+
+import argparse
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+
+from . import VERSION, detect_html, detect_response
+
+UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
+
+EPILOG = """examples:
+  captcha-detector https://www.google.com/recaptcha/api2/demo
+  captcha-detector https://example.com --browser --json
+  curl -s https://example.com | captcha-detector -
+  curl -s "https://scraper-api.example/v1?url=..." | captcha-detector -   # HTML or a scraper API's JSON
+"""
+
+
+def from_url(url, a):
+    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9"})
+    try:
+        res = urllib.request.urlopen(req, timeout=30)
+    except urllib.error.HTTPError as e:  # 403/429/503 pages are what we want to see
+        res = e
+    body = res.read()
+    charset = res.headers.get_content_charset() or "utf-8"
+    return detect_html(body.decode(charset, "replace"), url=res.geturl(), status=res.getcode(), headers=list(res.headers.items()), min_confidence=a.min)
+
+
+def from_browser(url, a):
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        raise SystemExit("captcha-detector: --browser needs Playwright: pip install playwright && playwright install chromium")
+    from .browser import detect_in_page
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=not a.headful)
+        try:
+            page = browser.new_context(viewport={"width": 1280, "height": 900}).new_page()
+            response = page.goto(url, wait_until="domcontentloaded", timeout=45000)
+            try:
+                page.wait_for_load_state("load", timeout=15000)
+            except Exception:
+                pass
+            if a.wait > 0:
+                page.wait_for_timeout(a.wait)
+            return detect_in_page(page, response=response, min_confidence=a.min)
+        finally:
+            browser.close()
+
+
+def show(r):
+    tty = sys.stdout.isatty()
+
+    def c(code, s):
+        return "\x1b[%sm%s\x1b[0m" % (code, s) if tty else s
+
+    head = c("33;1", "⚠ Captcha found") if r["captcha"] else c("31;1", "⛔ Blocked") if r["blocked"] else c("32;1", "✓ No captcha found")
+    print(head + ("  " + c("2", r["url"]) if r["url"] else ""))
+    if r["captcha"]:
+        vis = "unknown (static HTML)" if r["visible"] is None else str(r["visible"]).lower()
+        print(c("2", "  visible: %s   challenge page: %s" % (vis, str(r["challenge_page"]).lower())))
+    for d in r["detections"]:
+        bits = [d["vendor"], d["kind"] + ("/" + d["subtype"] if d["subtype"] else ""), "visible" if d["visible"] is True else "not visible" if d["visible"] is False else None, "%d%%" % int(d["confidence"] * 100 + 0.5)]
+        print("  • %s %s" % (c("1", d["name"]), c("2", "— " + " · ".join(b for b in bits if b))))
+        for e in d["evidence"][:4]:
+            print(c("2", "      %s: %s%s" % (e["type"], e["value"], " (in frame)" if e.get("frame") else "")))
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="captcha-detector", description="captcha-detector %s: does this page have a captcha?" % VERSION, epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("input", help="a URL, an HTML (or scraper API JSON) file, or - for stdin")
+    ap.add_argument("--browser", action="store_true", help="open the page in a real browser (needs: pip install playwright) and check the live page and its frames")
+    ap.add_argument("--headful", action="store_true", help="show the browser window (with --browser)")
+    ap.add_argument("--wait", type=int, default=3000, metavar="MS", help="wait after the page loads, with --browser (default 3000)")
+    ap.add_argument("--json", action="store_true", help="print the full result as JSON")
+    ap.add_argument("--fail", action="store_true", help="exit with code 1 when a captcha is found")
+    ap.add_argument("--min", type=float, default=0.5, metavar="0..1", help="minimum confidence (default 0.5)")
+    ap.add_argument("--version", action="version", version=VERSION)
+    a = ap.parse_args(argv)
+    try:
+        if a.input == "-":
+            r = detect_response(sys.stdin.buffer.read(), min_confidence=a.min)
+        elif re.match(r"^https?://", a.input, re.I):
+            r = from_browser(a.input, a) if a.browser else from_url(a.input, a)
+        elif os.path.exists(a.input):
+            with open(a.input, "rb") as f:
+                r = detect_response(f.read(), min_confidence=a.min)
+        elif re.match(r"^[\w-]+(\.[\w-]+)+", a.input):
+            url = "https://" + a.input
+            r = from_browser(url, a) if a.browser else from_url(url, a)
+        else:
+            raise ValueError("Not a URL or a file: " + a.input)
+    except SystemExit:
+        raise
+    except Exception as e:
+        print("captcha-detector: %s" % e, file=sys.stderr)
+        return 2
+    if a.json:
+        print(json.dumps(r, indent=2, ensure_ascii=False))
+    else:
+        show(r)
+    return 1 if a.fail and r["captcha"] else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

captcha_detector/browser.py

@@ -0,0 +1,231 @@
+"""Browser helpers: run the JavaScript build inside a real page (Playwright or Selenium), in every frame, and merge.
+
+In a browser the detector sees what scripts added after load and knows what is on screen, so `visible` is
+True/False instead of None. Cross-origin frames (reCAPTCHA, hCaptcha, Turnstile, DataDome) are checked from the
+outside, which page scripts cannot do.
+"""
+
+import json
+from pathlib import Path
+
+from .engine import from_js, merge_results
+
+#: Source of the browser build (exposes window.CaptchaDetector when loaded with a <script> tag).
+BROWSER_SCRIPT = (Path(__file__).parent / "captcha-detector.js").read_text(encoding="utf-8")
+
+MAX_FRAMES = 40
+
+_FRAME_VISIBLE = """(e) => {
+  const r = e.getBoundingClientRect();
+  const cs = getComputedStyle(e);
+  if (r.width < 10 || r.height < 10 || cs.display === 'none' || cs.visibility === 'hidden' || parseFloat(cs.opacity) < 0.05) return false;
+  if (e.checkVisibility && !e.checkVisibility({ opacityProperty: true, visibilityProperty: true })) return false;
+  return r.right + scrollX > 0 && r.bottom + scrollY > 0;
+}"""
+
+
+def _js_opts(min_confidence=None, deep=None, headers=None, status=None):
+    o = {}
+    if min_confidence is not None:
+        o["minConfidence"] = min_confidence
+    if deep is not None:
+        o["deep"] = deep
+    if headers:
+        o["headers"] = {str(k).lower(): str(v) for k, v in dict(headers).items()}
+    if status is not None:
+        o["status"] = status
+    return o
+
+
+def expression(min_confidence=None, deep=None, headers=None, status=None):
+    """A JavaScript expression that runs detect() in a page and returns the result, leaving no globals behind."""
+    opts = json.dumps(_js_opts(min_confidence, deep, headers, status))
+    return "(() => { const module = { exports: {} }; %s\n;const r = module.exports.detect(%s); return JSON.parse(JSON.stringify(r)); })()" % (BROWSER_SCRIPT, opts)
+
+
+def _skip(url):
+    return not url or url == "about:blank" or url.startswith("about:srcdoc")
+
+
+# ───────────────────────────── Playwright (sync) ─────────────────────────────
+
+
+def _pw_response(response):
+    if response is None:
+        return {}
+    try:
+        h = response.all_headers() if hasattr(response, "all_headers") else response.headers
+        return {"status": response.status, "headers": h}
+    except Exception:
+        return {}
+
+
+def _pw_frame_visible(frame):
+    try:
+        el = frame.frame_element()
+        if not el or not el.evaluate(_FRAME_VISIBLE):
+            return False
+        parent = frame.parent_frame
+        if parent is not None and parent.parent_frame is not None:
+            return _pw_frame_visible(parent)
+        return True
+    except Exception:
+        return None
+
+
+def detect_in_page(page, response=None, frames=True, min_confidence=None, deep=None):
+    """Detect captchas on a Playwright page (sync API), inside its frames too.
+
+    response  the Response from page.goto(), so its status and headers are checked too
+    frames    also look inside every frame, cross-origin ones included (default True)
+    """
+    info = _pw_response(response)
+    main_frame = page.main_frame
+    main = main_frame.evaluate(expression(min_confidence, deep, info.get("headers"), info.get("status")))
+    if not frames:
+        return from_js(main)
+    results = []
+    for frame in [f for f in page.frames if f != main_frame][:MAX_FRAMES]:
+        url = frame.url
+        if _skip(url):
+            continue
+        try:
+            results.append({"url": url, "visible": _pw_frame_visible(frame), "result": frame.evaluate(expression(min_confidence, deep))})
+        except Exception:
+            pass  # detached or not ready
+    return merge_results(main, results)
+
+
+def wait_for_captcha(page, timeout=10.0, interval=0.5, visible=False, **kw):
+    """Poll until a captcha appears (or with visible=True: until one is on screen), or `timeout` seconds pass.
+    Returns the last result."""
+    import time
+
+    end = time.monotonic() + timeout
+    while True:
+        last = detect_in_page(page, **kw)
+        if last["captcha"] and (not visible or last["visible"]):
+            return last
+        if time.monotonic() >= end:
+            return last
+        page.wait_for_timeout(interval * 1000)
+
+
+# ───────────────────────────── Playwright (async) ─────────────────────────────
+
+
+async def _pw_response_async(response):
+    if response is None:
+        return {}
+    try:
+        h = await response.all_headers() if hasattr(response, "all_headers") else response.headers
+        return {"status": response.status, "headers": h}
+    except Exception:
+        return {}
+
+
+async def _pw_frame_visible_async(frame):
+    try:
+        el = await frame.frame_element()
+        if not el or not await el.evaluate(_FRAME_VISIBLE):
+            return False
+        parent = frame.parent_frame
+        if parent is not None and parent.parent_frame is not None:
+            return await _pw_frame_visible_async(parent)
+        return True
+    except Exception:
+        return None
+
+
+async def detect_in_page_async(page, response=None, frames=True, min_confidence=None, deep=None):
+    """detect_in_page() for Playwright's async API."""
+    import asyncio
+
+    info = await _pw_response_async(response)
+    main_frame = page.main_frame
+    main = await main_frame.evaluate(expression(min_confidence, deep, info.get("headers"), info.get("status")))
+    if not frames:
+        return from_js(main)
+
+    async def one(frame):
+        url = frame.url
+        if _skip(url):
+            return None
+        try:
+            result, vis = await asyncio.gather(frame.evaluate(expression(min_confidence, deep)), _pw_frame_visible_async(frame))
+            return {"url": url, "visible": vis, "result": result}
+        except Exception:
+            return None
+
+    found = await asyncio.gather(*[one(f) for f in [f for f in page.frames if f != main_frame][:MAX_FRAMES]])
+    return merge_results(main, [r for r in found if r])
+
+
+async def wait_for_captcha_async(page, timeout=10.0, interval=0.5, visible=False, **kw):
+    """wait_for_captcha() for Playwright's async API."""
+    import asyncio
+    import time
+
+    end = time.monotonic() + timeout
+    while True:
+        last = await detect_in_page_async(page, **kw)
+        if last["captcha"] and (not visible or last["visible"]):
+            return last
+        if time.monotonic() >= end:
+            return last
+        await asyncio.sleep(interval)
+
+
+# ───────────────────────────── Selenium ─────────────────────────────
+
+_SEL_FRAMES = "return Array.from(document.querySelectorAll('iframe, frame'))"
+_SEL_VISIBLE = "return (%s)(arguments[0])" % _FRAME_VISIBLE
+
+
+def detect_selenium(driver, frames=True, min_confidence=None, deep=None, max_depth=2):
+    """Detect captchas with a Selenium WebDriver, inside its frames too (cross-origin ones included).
+
+    Switches into each frame and back; the driver is left on the top-level page.
+    """
+    expr = "return " + expression(min_confidence, deep)
+    driver.switch_to.default_content()
+    main = driver.execute_script(expr)
+    if not frames:
+        return from_js(main)
+    results = []
+
+    def walk(path, parent_visible, depth):
+        try:
+            els = driver.execute_script(_SEL_FRAMES) or []
+        except Exception:
+            return
+        for i in range(len(els)):
+            if len(results) >= MAX_FRAMES:
+                return
+            try:
+                els = driver.execute_script(_SEL_FRAMES) or []  # fresh handles after switching back
+                el = els[i]
+                url = el.get_attribute("src") or ""
+                vis = bool(driver.execute_script(_SEL_VISIBLE, el)) and parent_visible
+                driver.switch_to.frame(el)
+                try:
+                    url = driver.execute_script("return location.href") or url
+                    if not _skip(url):
+                        results.append({"url": url, "visible": vis, "result": driver.execute_script(expr)})
+                    if depth < max_depth:
+                        walk(path + [i], vis, depth + 1)
+                finally:
+                    driver.switch_to.default_content()
+                    for j in path:
+                        driver.switch_to.frame(driver.execute_script(_SEL_FRAMES)[j])
+            except Exception:
+                try:
+                    driver.switch_to.default_content()
+                    for j in path:
+                        driver.switch_to.frame(driver.execute_script(_SEL_FRAMES)[j])
+                except Exception:
+                    return
+
+    walk([], True, 1)
+    driver.switch_to.default_content()
+    return merge_results(main, results)