linksanity 0.1.0__py3-none-any.whl

linksanity/__init__.py

@@ -0,0 +1,3 @@
+"""linksanity — detect broken links in Markdown, reStructuredText, and HTML documentation."""
+
+__version__ = "0.1.0"

linksanity/__main__.py

@@ -0,0 +1,3 @@
+from linksanity.cli import app
+
+app()

linksanity/checkers/filesystem.py

@@ -0,0 +1,136 @@
+"""Check internal links and anchor fragments against the local filesystem."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from linksanity.queue import LinkResult, LinkStatus, LinkType
+
+
+def check(
+    url: str,
+    source_file: str,
+    line: int,
+    link_type: LinkType,
+    *,
+    check_anchors: bool = False,
+) -> LinkResult:
+    """Resolve and validate an internal or anchor link.
+
+    For ANCHOR links (#fragment), validates the fragment exists in the source
+    file when check_anchors is True.
+    For INTERNAL links (./path or ../path), validates the target file exists.
+    """
+    source_path = Path(source_file)
+
+    # Split path and fragment
+    fragment: str | None = None
+    path_part = url
+    if "#" in url:
+        path_part, fragment = url.split("#", 1)
+
+    # Resolve the target file
+    if link_type == LinkType.ANCHOR or not path_part:
+        target_path = source_path
+    else:
+        target_path = (source_path.parent / path_part).resolve()
+
+    # Check file existence for non-pure-anchor links
+    if path_part and not target_path.exists():
+        return LinkResult(
+            source_file=source_file,
+            line=line,
+            url=url,
+            link_type=link_type,
+            status=LinkStatus.BROKEN,
+            error=f"file not found: {target_path}",
+        )
+
+    # Optionally validate anchor fragment
+    if fragment and check_anchors and not _anchor_exists(target_path, fragment):
+        return LinkResult(
+            source_file=source_file,
+            line=line,
+            url=url,
+            link_type=link_type,
+            status=LinkStatus.BROKEN,
+            error=f"anchor '#{fragment}' not found in {target_path.name}",
+        )
+
+    return LinkResult(
+        source_file=source_file,
+        line=line,
+        url=url,
+        link_type=link_type,
+        status=LinkStatus.OK,
+    )
+
+
+def _anchor_exists(path: Path, fragment: str) -> bool:
+    """Return True if fragment matches a heading/ID in the file."""
+    suffix = path.suffix.lower()
+    try:
+        content = path.read_text(encoding="utf-8")
+    except OSError:
+        return False
+
+    if suffix == ".md":
+        return fragment in _md_anchors(content)
+    if suffix == ".rst":
+        return fragment in _rst_anchors(content)
+    if suffix in (".html", ".htm"):
+        return fragment in _html_ids(content)
+    return False
+
+
+def _md_anchors(content: str) -> set[str]:
+    """Extract GitHub-style anchor slugs from Markdown headings."""
+    anchors: set[str] = set()
+    for line in content.splitlines():
+        # ATX headings: # Heading, ## Heading, etc.
+        m = re.match(r"^#{1,6}\s+(.+?)(?:\s+#+)?$", line)
+        if m:
+            anchors.add(_gh_slug(m.group(1)))
+    return anchors
+
+
+def _gh_slug(text: str) -> str:
+    """Convert heading text to a GitHub Markdown anchor slug."""
+    text = text.lower()
+    text = re.sub(r"[^\w\s-]", "", text)   # remove special chars except - and _
+    text = re.sub(r"\s+", "-", text.strip())
+    return text
+
+
+def _rst_anchors(content: str) -> set[str]:
+    """Extract docutils-style IDs from all RST nodes that carry ids."""
+    from io import StringIO
+
+    from docutils.core import publish_doctree
+    from docutils.utils import Reporter
+
+    anchors: set[str] = set()
+    try:
+        doc = publish_doctree(
+            content,
+            settings_overrides={
+                "report_level": Reporter.SEVERE_LEVEL,
+                "halt_level": Reporter.SEVERE_LEVEL,
+                "warning_stream": StringIO(),
+            },
+        )
+        # Walk every Element node — titles, sections, and targets all carry ids
+        from docutils.nodes import Element
+        for node in doc.findall(Element):
+            for id_ in node.get("ids", []):
+                if isinstance(id_, str):
+                    anchors.add(id_)
+    except Exception:  # noqa: BLE001
+        pass
+    return anchors
+
+
+def _html_ids(content: str) -> set[str]:
+    """Extract all id= attribute values from HTML."""
+    return set(re.findall(r'\bid=["\']([^"\']+)["\']', content))

linksanity/checkers/http.py

@@ -0,0 +1,171 @@
+"""Async HTTP link checker using httpx with retry and fallback."""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+from urllib.parse import urlparse
+
+import httpx
+
+from linksanity.queue import LinkResult, LinkStatus, LinkType
+
+_RETRY_ON = {429, 503}
+_FALLBACK_ON = {405}
+_TIMEOUT = httpx.Timeout(10.0)
+_HEADERS = {"User-Agent": "linksanity/0.1 link-checker (+https://github.com/linksanity)"}
+
+# Hostnames that are always private regardless of DNS resolution
+_PRIVATE_HOSTNAMES = frozenset({"localhost", "metadata.google.internal"})
+
+
+def _is_private_host(hostname: str) -> bool:
+    """Return True if hostname is a loopback, link-local, or private address."""
+    if hostname.lower() in _PRIVATE_HOSTNAMES:
+        return True
+    try:
+        addr = ipaddress.ip_address(hostname)
+        return addr.is_loopback or addr.is_link_local or addr.is_private
+    except ValueError:
+        return False
+
+
+async def check(
+    url: str,
+    source_file: str,
+    line: int,
+    link_type: LinkType,
+    *,
+    ignore_domains: set[str] | None = None,
+    timeout: int = 10,
+    retries: int = 2,
+) -> LinkResult:
+    """Check an external URL and return a LinkResult.
+
+    Strategy:
+    1. HEAD request first (fast, low bandwidth).
+    2. On 405 Method Not Allowed, retry with GET + stream (no body download).
+    3. On 429/503, retry up to `retries` times with exponential backoff.
+    """
+    parsed = urlparse(url)
+    domain = parsed.netloc.lower()
+    hostname = parsed.hostname or ""
+
+    if _is_private_host(hostname):
+        return LinkResult(
+            source_file=source_file, line=line, url=url,
+            link_type=link_type, status=LinkStatus.SKIPPED,
+            error="skipped: private/loopback address",
+        )
+
+    if ignore_domains and _domain_match(domain, ignore_domains):
+        return LinkResult(
+            source_file=source_file, line=line, url=url,
+            link_type=link_type, status=LinkStatus.SKIPPED,
+        )
+
+    client_timeout = httpx.Timeout(float(timeout))
+    try:
+        async with httpx.AsyncClient(
+            follow_redirects=True,
+            timeout=client_timeout,
+            headers=_HEADERS,
+        ) as client:
+            return await _check_with_retry(
+                client, url, source_file, line, link_type, retries
+            )
+    except Exception as exc:
+        return LinkResult(
+            source_file=source_file, line=line, url=url,
+            link_type=link_type, status=LinkStatus.ERROR,
+            error=str(exc),
+        )
+
+
+async def _check_with_retry(
+    client: httpx.AsyncClient,
+    url: str,
+    source_file: str,
+    line: int,
+    link_type: LinkType,
+    retries: int,
+) -> LinkResult:
+    last_exc: Exception | None = None
+    for attempt in range(retries + 1):
+        try:
+            result = await _try_head(client, url, source_file, line, link_type)
+            if result.http_code in _RETRY_ON and attempt < retries:
+                await asyncio.sleep(2 ** attempt)
+                continue
+            return result
+        except httpx.HTTPError as exc:
+            last_exc = exc
+            if attempt < retries:
+                await asyncio.sleep(2 ** attempt)
+
+    return LinkResult(
+        source_file=source_file, line=line, url=url,
+        link_type=link_type, status=LinkStatus.ERROR,
+        error=str(last_exc),
+    )
+
+
+async def _try_head(
+    client: httpx.AsyncClient,
+    url: str,
+    source_file: str,
+    line: int,
+    link_type: LinkType,
+) -> LinkResult:
+    try:
+        resp = await client.head(url)
+    except httpx.HTTPError:
+        raise
+
+    if resp.status_code in _FALLBACK_ON:
+        # Server doesn't support HEAD — try GET with streaming (no body)
+        async with client.stream("GET", url) as stream_resp:
+            code = stream_resp.status_code
+            resolved = str(stream_resp.url)
+    else:
+        code = resp.status_code
+        resolved = str(resp.url)
+
+    return _make_result(url, source_file, line, link_type, code, resolved)
+
+
+def _make_result(
+    url: str,
+    source_file: str,
+    line: int,
+    link_type: LinkType,
+    code: int,
+    resolved_url: str,
+) -> LinkResult:
+    # With follow_redirects=True, httpx resolves the full chain.
+    # A redirect is detected when the final URL differs from the original.
+    was_redirected = resolved_url.rstrip("/") != url.rstrip("/")
+
+    if code >= 400:
+        status = LinkStatus.BROKEN
+    elif was_redirected:
+        status = LinkStatus.REDIRECT
+    else:
+        status = LinkStatus.OK
+
+    return LinkResult(
+        source_file=source_file,
+        line=line,
+        url=url,
+        link_type=link_type,
+        status=status,
+        http_code=code,
+        resolved_url=resolved_url if was_redirected else None,
+    )
+
+
+def _domain_match(domain: str, ignore_set: set[str]) -> bool:
+    """Return True if domain or any parent domain is in the ignore set."""
+    return domain in ignore_set or any(
+        domain.endswith("." + d) for d in ignore_set
+    )

linksanity/checkers/playwright.py

@@ -0,0 +1,228 @@
+"""Playwright-based link extractor and checker for JS-rendered pages."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from urllib.parse import urlparse
+
+from linksanity.queue import LinkResult, LinkStatus, LinkType
+
+_SKIP_SCHEMES = ("mailto:", "javascript:", "data:", "blob:")
+
+# Well-known analytics and tracking domains. Requests to these are aborted
+# when --block-analytics is set, speeding up crawls and suppressing false hits.
+ANALYTICS_DOMAINS: frozenset[str] = frozenset({
+    "google-analytics.com",
+    "analytics.google.com",
+    "googletagmanager.com",
+    "googletagservices.com",
+    "doubleclick.net",
+    "hotjar.com",
+    "segment.com",
+    "cdn.segment.com",
+    "api.segment.io",
+    "mixpanel.com",
+    "amplitude.com",
+    "heap.io",
+    "heapanalytics.com",
+    "fullstory.com",
+    "clarity.ms",
+    "plausible.io",
+    "intercom.io",
+    "intercomcdn.com",
+    "widget.intercom.io",
+})
+
+
+def _require_playwright() -> None:
+    try:
+        import playwright  # noqa: F401
+    except ImportError:
+        raise ImportError(
+            "Playwright is not installed. "
+            "Run: pip install linksanity[browser] && playwright install chromium"
+        ) from None
+
+
+async def extract_links(url: str, *, semaphore: asyncio.Semaphore | None = None) -> list[str]:
+    """Launch a headless browser, render the page, and return all href values.
+
+    Filters out mailto:, javascript:, data:, blob:, and empty hrefs.
+    semaphore limits concurrent browser contexts.
+    """
+    _require_playwright()
+    from playwright.async_api import async_playwright
+
+    sem = semaphore or asyncio.Semaphore(2)
+    async with sem, async_playwright() as pw:
+        browser = await pw.chromium.launch(headless=True)
+        try:
+            page = await browser.new_page()
+            await page.goto(url, wait_until="domcontentloaded", timeout=30_000)
+            hrefs: list[str] = await page.eval_on_selector_all(
+                "a[href]",
+                "els => els.map(e => e.href).filter(h => h)",
+            )
+            return [
+                h for h in hrefs
+                if h and not any(h.startswith(s) for s in _SKIP_SCHEMES)
+            ]
+        finally:
+            await browser.close()
+
+
+async def check(
+    url: str,
+    source_file: str,
+    line: int,
+    link_type: LinkType,
+    *,
+    semaphore: asyncio.Semaphore | None = None,
+    timeout: int = 10,
+) -> LinkResult:
+    """Check whether a URL is reachable using a headless browser.
+
+    Uses Playwright's network response to determine status.
+    """
+    _require_playwright()
+    from playwright.async_api import Error as PlaywrightError
+    from playwright.async_api import async_playwright
+
+    sem = semaphore or asyncio.Semaphore(2)
+    async with sem, async_playwright() as pw:
+        browser = await pw.chromium.launch(headless=True)
+        try:
+            page = await browser.new_page()
+            try:
+                response = await page.goto(
+                    url,
+                    wait_until="domcontentloaded",
+                    timeout=timeout * 1000,
+                )
+                if response is None:
+                    return LinkResult(
+                        source_file=source_file, line=line, url=url,
+                        link_type=link_type, status=LinkStatus.ERROR,
+                        error="no response",
+                    )
+                code = response.status
+                resolved = page.url
+                was_redirected = _strip(resolved) != _strip(url)
+                if code >= 400:
+                    status = LinkStatus.BROKEN
+                elif was_redirected:
+                    status = LinkStatus.REDIRECT
+                else:
+                    status = LinkStatus.OK
+                return LinkResult(
+                    source_file=source_file, line=line, url=url,
+                    link_type=link_type, status=status,
+                    http_code=code,
+                    resolved_url=resolved if was_redirected else None,
+                )
+            except PlaywrightError as exc:
+                return LinkResult(
+                    source_file=source_file, line=line, url=url,
+                    link_type=link_type, status=LinkStatus.ERROR,
+                    error=str(exc),
+                )
+        finally:
+            await browser.close()
+
+
+async def crawl_page(
+    url: str,
+    source_file: str,
+    line: int,
+    link_type: LinkType,
+    *,
+    semaphore: asyncio.Semaphore | None = None,
+    timeout: int = 10,
+    block_domains: frozenset[str] | None = None,
+) -> tuple[LinkResult, list[str]]:
+    """Visit a page, check its reachability, and return (result, hrefs).
+
+    Combines check() and extract_links() into a single browser session.
+    """
+    _require_playwright()
+    from playwright.async_api import Error as PlaywrightError
+    from playwright.async_api import async_playwright
+
+    sem = semaphore or asyncio.Semaphore(2)
+    async with sem, async_playwright() as pw:
+        browser = await pw.chromium.launch(headless=True)
+        try:
+            page = await browser.new_page()
+            if block_domains:
+                from playwright.async_api import Route
+
+                async def _block(route: Route) -> None:
+                    netloc = urlparse(route.request.url).netloc.lower()
+                    if any(netloc == d or netloc.endswith("." + d) for d in block_domains):
+                        await route.abort()
+                    else:
+                        await route.continue_()
+
+                await page.route("**/*", _block)
+            try:
+                response = await page.goto(
+                    url,
+                    wait_until="domcontentloaded",
+                    timeout=timeout * 1000,
+                )
+                if response is None:
+                    return LinkResult(
+                        source_file=source_file, line=line, url=url,
+                        link_type=link_type, status=LinkStatus.ERROR,
+                        error="no response",
+                    ), []
+                # Wait for JS to finish rendering navigation (SPAs build links client-side)
+                with contextlib.suppress(PlaywrightError):
+                    await page.wait_for_load_state("networkidle", timeout=5000)
+                code = response.status
+                resolved = page.url
+                was_redirected = _strip(resolved) != _strip(url)
+                if code >= 400:
+                    status = LinkStatus.BROKEN
+                elif was_redirected:
+                    status = LinkStatus.REDIRECT
+                else:
+                    status = LinkStatus.OK
+                result = LinkResult(
+                    source_file=source_file, line=line, url=url,
+                    link_type=link_type, status=status,
+                    http_code=code,
+                    resolved_url=resolved if was_redirected else None,
+                )
+                # Extract links from any reachable page, including redirects
+                if status in (LinkStatus.OK, LinkStatus.REDIRECT):
+                    hrefs: list[str] = await page.eval_on_selector_all(
+                        "a[href]",
+                        "els => els.map(e => e.href).filter(h => h)",
+                    )
+                    links = [
+                        h for h in hrefs
+                        if h and not any(h.startswith(s) for s in _SKIP_SCHEMES)
+                    ]
+                else:
+                    links = []
+                return result, links
+            except PlaywrightError as exc:
+                return LinkResult(
+                    source_file=source_file, line=line, url=url,
+                    link_type=link_type, status=LinkStatus.ERROR,
+                    error=str(exc),
+                ), []
+        finally:
+            await browser.close()
+
+
+def _strip(url: str) -> str:
+    return url.rstrip("/")
+
+
+def scope_filter(urls: list[str], start_url: str) -> list[str]:
+    """Keep only URLs on the same domain as start_url."""
+    domain = urlparse(start_url).netloc.lower()
+    return [u for u in urls if urlparse(u).netloc.lower() == domain]