crawlableseo 0.1.0__py3-none-any.whl

crawlableseo/__init__.py

@@ -0,0 +1,42 @@
+"""Make a client-rendered SPA crawlable from its own HTML shell.
+
+No headless browser, no SSR framework, no third-party rendering service:
+the server fills in the shell's ``<head>`` and mount node per URL, and the
+same declaration produces robots.txt, sitemap.xml and llms.txt.
+"""
+
+from __future__ import annotations
+
+from .head import head_block
+from .indexnow import IndexNow
+from .llmstxt import LlmsSection, llms_txt
+from .page import MAX_DESCRIPTION, MAX_TITLE, MIN_DESCRIPTION, NotFound, Page
+from .robots import DEFAULT_AI_CRAWLERS, robots_txt
+from .shell import MARKER, render_shell
+from .site import DynamicUrl, Site
+from .sitemap import SitemapEntry, sitemap_xml
+from .urls import page_url
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "DEFAULT_AI_CRAWLERS",
+    "MARKER",
+    "MAX_DESCRIPTION",
+    "MAX_TITLE",
+    "MIN_DESCRIPTION",
+    "DynamicUrl",
+    "IndexNow",
+    "LlmsSection",
+    "NotFound",
+    "Page",
+    "Site",
+    "SitemapEntry",
+    "__version__",
+    "head_block",
+    "llms_txt",
+    "page_url",
+    "render_shell",
+    "robots_txt",
+    "sitemap_xml",
+]

crawlableseo/head.py

@@ -0,0 +1,58 @@
+"""The block of tags written into <head> for one page."""
+
+from __future__ import annotations
+
+import html
+import json
+
+from .page import Page
+from .shell_marker import MARKER
+
+
+def head_block(
+    page: Page,
+    *,
+    canonical: str,
+    base_url: str,
+    site_name: str = "",
+    twitter_site: str = "",
+    default_og_image: str | None = None,
+) -> str:
+    e = html.escape
+    img = page.og_image or default_og_image
+    if img and img.startswith("/"):
+        img = base_url.rstrip("/") + img
+
+    lines = [
+        MARKER,
+        f'<link rel="canonical" href="{e(canonical)}" />',
+        f'<meta name="robots" content="{page.robots_value()}" />',
+        '<meta property="og:type" content="website" />',
+        f'<meta property="og:title" content="{e(page.title)}" />',
+        f'<meta property="og:description" content="{e(page.description)}" />',
+        f'<meta property="og:url" content="{e(canonical)}" />',
+        '<meta name="twitter:card" content="summary_large_image" />',
+        f'<meta name="twitter:title" content="{e(page.title)}" />',
+        f'<meta name="twitter:description" content="{e(page.description)}" />',
+    ]
+    if site_name:
+        lines.insert(4, f'<meta property="og:site_name" content="{e(site_name)}" />')
+    if twitter_site:
+        lines.append(f'<meta name="twitter:site" content="{e(twitter_site)}" />')
+    if img:
+        lines += [
+            f'<meta property="og:image" content="{e(img)}" />',
+            '<meta property="og:image:width" content="1200" />',
+            '<meta property="og:image:height" content="630" />',
+            f'<meta name="twitter:image" content="{e(img)}" />',
+        ]
+    lines += [_jsonld_tag(obj) for obj in page.jsonld]
+    return "\n".join(lines)
+
+
+def _jsonld_tag(obj: dict[str, object]) -> str:
+    # A "</" anywhere in the data would close the script element early, so the
+    # rest of the document becomes the browser's problem. Escaping the slash
+    # is valid JSON and parses back to the same string.
+    payload = json.dumps(obj, separators=(",", ":"), ensure_ascii=False).replace("</", "<\\/")
+    return '<script type="application/ld+json">' + payload + "</script>"

crawlableseo/indexnow.py

@@ -0,0 +1,111 @@
+"""IndexNow: tell search engines a URL changed, instead of waiting to be crawled.
+
+The protocol is small. The site publishes a key at ``/<key>.txt`` whose body
+is the key itself, then POSTs changed URLs with that key. Bing, Yandex,
+Seznam and Naver share one endpoint; Bing matters most because it feeds
+ChatGPT search and Copilot.
+
+The key is public by design - anyone can read the key file - and grants
+nothing beyond submitting URLs for that one host.
+
+IndexNow is an accelerator, never a dependency: engines re-crawl on their own
+schedule regardless, so a failed submission is logged and dropped, never
+raised into the caller's request.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Sequence
+
+logger = logging.getLogger("crawlableseo.indexnow")
+
+ENDPOINT = "https://api.indexnow.org/indexnow"
+# The protocol caps one submission at 10,000 URLs. Batching well below that
+# keeps a whole-sitemap resubmit to a handful of requests.
+BATCH = 500
+_KEY_RE = re.compile(r"^[A-Za-z0-9-]{8,128}$")
+
+
+class IndexNow:
+    """Client for one host.
+
+    ``httpx`` is an optional dependency; install ``crawlableseo[indexnow]``
+    to use this class.
+    """
+
+    def __init__(self, base_url: str, key: str | None, *, endpoint: str = ENDPOINT) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.endpoint = endpoint
+        self.key = self._validated(key)
+
+    @staticmethod
+    def _validated(key: str | None) -> str | None:
+        k = (key or "").strip()
+        if not k:
+            return None
+        if not _KEY_RE.match(k):
+            logger.warning(
+                "indexnow: key must be 8-128 characters of [A-Za-z0-9-]; disabled"
+            )
+            return None
+        return k
+
+    @property
+    def enabled(self) -> bool:
+        return self.key is not None
+
+    @property
+    def key_path(self) -> str | None:
+        """Path the key file must be served at, e.g. ``/abc123.txt``."""
+        return f"/{self.key}.txt" if self.key else None
+
+    def absolute(self, urls: Sequence[str]) -> list[str]:
+        out = []
+        for u in urls:
+            if not u:
+                continue
+            out.append(u if u.startswith("http") else self.base_url + u)
+        return out
+
+    async def submit(self, urls: Sequence[str]) -> bool:
+        """POST the URLs. Returns False when disabled or when the call failed."""
+        if not self.key:
+            logger.debug("indexnow: no key configured, skipping %d url(s)", len(urls))
+            return False
+        try:
+            import httpx
+        except ImportError:  # pragma: no cover - exercised by the extras test
+            logger.warning("indexnow: httpx is not installed; install crawlableseo[indexnow]")
+            return False
+
+        host = self.base_url.split("://", 1)[-1].split("/", 1)[0]
+        absolute = self.absolute(urls)
+        ok = True
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            for start in range(0, len(absolute), BATCH):
+                chunk = absolute[start : start + BATCH]
+                payload = {
+                    "host": host,
+                    "key": self.key,
+                    "keyLocation": f"{self.base_url}{self.key_path}",
+                    "urlList": chunk,
+                }
+                try:
+                    response = await client.post(self.endpoint, json=payload)
+                except Exception as exc:
+                    logger.warning("indexnow: submission failed (%s)", exc)
+                    ok = False
+                    continue
+                if response.status_code >= 400:
+                    logger.warning(
+                        "indexnow: %s rejected %d url(s): %s",
+                        response.status_code,
+                        len(chunk),
+                        response.text[:200],
+                    )
+                    ok = False
+                else:
+                    logger.info("indexnow: submitted %d url(s)", len(chunk))
+        return ok

crawlableseo/integrations/fastapi.py

@@ -0,0 +1,83 @@
+"""Wire a :class:`~crawlableseo.site.Site` into FastAPI or Starlette.
+
+``router(site)`` returns the crawler-facing endpoints plus the SPA catch-all.
+Mount it last: the catch-all answers every path the rest of the app did not
+claim.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+
+# Imported at module scope on purpose. FastAPI resolves a handler's
+# annotations against its module's globals, so a Request imported inside the
+# function is invisible to it: every request then fails validation with
+# "query.request field required" instead of being served. This module is only
+# imported by callers who already depend on FastAPI.
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import HTMLResponse, PlainTextResponse
+
+from ..indexnow import IndexNow
+from ..llmstxt import LlmsSection
+from ..robots import DEFAULT_AI_CRAWLERS
+from ..site import Site
+
+
+def router(
+    site: Site,
+    *,
+    disallow: Sequence[str] = (),
+    ai_crawlers: Sequence[str] | None = DEFAULT_AI_CRAWLERS,
+    llms: Callable[[], str] | None = None,
+    indexnow: IndexNow | None = None,
+    shell_cache_seconds: int = 0,
+    text_cache_seconds: int = 3600,
+    catch_all: bool = True,
+) -> APIRouter:
+    """Build an ``APIRouter`` serving robots.txt, sitemap.xml, llms.txt and the SPA."""
+    api = APIRouter()
+    text_headers = {"Cache-Control": f"public, max-age={text_cache_seconds}"}
+
+    @api.get("/robots.txt", include_in_schema=False)
+    async def robots() -> Response:
+        return PlainTextResponse(
+            site.robots(disallow=disallow, ai_crawlers=ai_crawlers), headers=text_headers
+        )
+
+    @api.get("/sitemap.xml", include_in_schema=False)
+    async def sitemap() -> Response:
+        return Response(
+            site.sitemap(), media_type="application/xml", headers=text_headers
+        )
+
+    if llms is not None:
+        @api.get("/llms.txt", include_in_schema=False)
+        async def llms_route() -> Response:
+            return PlainTextResponse(llms(), headers=text_headers)
+
+    if indexnow is not None and indexnow.key_path:
+        key = indexnow.key
+
+        @api.get(indexnow.key_path, include_in_schema=False)
+        async def indexnow_key() -> Response:
+            # The body must be exactly the key: that is how the engine
+            # verifies the submitter controls this host.
+            return PlainTextResponse(
+                key or "", headers={"Cache-Control": "public, max-age=86400"}
+            )
+
+    if catch_all:
+        @api.get("/{path:path}", include_in_schema=False)
+        async def spa(path: str, request: Request) -> Response:
+            html, status = site.render("/" + path, dict(request.query_params))
+            headers = (
+                {"Cache-Control": f"public, max-age={shell_cache_seconds}"}
+                if shell_cache_seconds
+                else {"Cache-Control": "no-cache"}
+            )
+            return HTMLResponse(html, status_code=status, headers=headers)
+
+    return api
+
+
+__all__ = ["LlmsSection", "router"]

crawlableseo/llmstxt.py

@@ -0,0 +1,38 @@
+"""llms.txt: a plain-language map of the site for models and agents.
+
+The format is Markdown by convention: an H1 with the site's name, a
+blockquote summarising it in one paragraph, then sections of links with a
+sentence each. Keep it honest and specific; a model quoting this file will
+quote whatever it says, including the parts that oversell.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class LlmsSection:
+    title: str
+    links: Sequence[tuple[str, str, str]]
+    """``(label, url, one-line description)`` per link."""
+
+
+def llms_txt(
+    name: str,
+    summary: str,
+    sections: Iterable[LlmsSection],
+    *,
+    notes: Sequence[str] = (),
+) -> str:
+    out = [f"# {name}", "", f"> {summary}", ""]
+    for note in notes:
+        out += [note, ""]
+    for section in sections:
+        out.append(f"## {section.title}")
+        out.append("")
+        for label, url, description in section.links:
+            out.append(f"- [{label}]({url}): {description}")
+        out.append("")
+    return "\n".join(out).rstrip() + "\n"

crawlableseo/page.py

@@ -0,0 +1,82 @@
+"""The one description of a page, shared by every output the library makes."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, replace
+
+# Bing's site scanner reports a description outside this range as an SEO
+# error, and Google truncates at roughly the same point.
+MIN_DESCRIPTION = 50
+MAX_DESCRIPTION = 160
+# Google renders about this much of a title before the ellipsis.
+MAX_TITLE = 60
+
+
+@dataclass(frozen=True)
+class Page:
+    """Everything the shell, the sitemap and llms.txt need about one URL.
+
+    ``body`` is crawlable HTML placed inside the mount node. The framework
+    replaces the node's children when it mounts, so users never see it; a
+    crawler that does not run JavaScript reads it as the page's content.
+    """
+
+    title: str
+    description: str
+    body: str = ""
+    # Extra query parameters that belong in the canonical URL. A parameter
+    # that changes the content belongs here; a tracking parameter does not.
+    params: dict[str, str] = field(default_factory=dict)
+    jsonld: tuple[dict[str, object], ...] = ()
+    og_image: str | None = None
+    index: bool = True
+    # Kept separate from ``index``: a page can be worth excluding from the
+    # index while its outgoing links are still worth crawling.
+    follow: bool = True
+    status: int = 200
+    # Sitemap hints. ``in_sitemap=False`` keeps a real page out of it.
+    in_sitemap: bool = True
+    changefreq: str = "weekly"
+    priority: float = 0.5
+    lastmod: str | None = None
+
+    def robots_value(self) -> str:
+        if self.index:
+            return "index, follow" if self.follow else "index, nofollow"
+        return "noindex, follow" if self.follow else "noindex, nofollow"
+
+    def clipped(self) -> Page:
+        """Title and description cut to the lengths search engines keep.
+
+        Applied once, where the tags are written, rather than trusted to
+        every caller: pages built from database strings have no length limit
+        of their own.
+        """
+        return replace(
+            self,
+            title=_clip(self.title, MAX_TITLE),
+            description=_clip(self.description, MAX_DESCRIPTION),
+        )
+
+
+def _clip(text: str, limit: int) -> str:
+    text = " ".join(text.split())
+    if len(text) <= limit:
+        return text
+    cut = text[: limit - 1].rstrip()
+    space = cut.rfind(" ")
+    if space > limit // 2:
+        cut = cut[:space]
+    return cut.rstrip(" ,.;:-") + "…"
+
+
+@dataclass(frozen=True)
+class NotFound:
+    """Returned by a resolver for a URL that looks valid but has no content.
+
+    Answering 200 with an "isn't available" screen is what search engines
+    file as a soft 404: the page is counted, judged empty, and the crawl
+    budget spent on it is gone. Say 404 instead.
+    """
+
+    canonical_to: str = "/"

crawlableseo/robots.py

@@ -0,0 +1,60 @@
+"""robots.txt, with the two rules that cost the most to learn."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+
+# Crawlers that feed AI answers. Being quotable by them is a distribution
+# channel, and the crawlable body this library injects is written for them.
+DEFAULT_AI_CRAWLERS: tuple[str, ...] = (
+    "GPTBot",
+    "OAI-SearchBot",
+    "ChatGPT-User",
+    "PerplexityBot",
+    "ClaudeBot",
+    "Claude-SearchBot",
+    "Google-Extended",
+)
+
+
+def robots_txt(
+    base_url: str,
+    *,
+    disallow: Sequence[str] = (),
+    allow: Sequence[str] = ("/",),
+    sitemap: str | Iterable[str] | None = "/sitemap.xml",
+    ai_crawlers: Sequence[str] | None = DEFAULT_AI_CRAWLERS,
+    extra_lines: Sequence[str] = (),
+) -> str:
+    """Build robots.txt.
+
+    Two warnings, both of them expensive in practice:
+
+    Do not disallow the JSON your own app fetches. A modern crawler renders
+    the page like a browser; blocking the API it calls leaves every route
+    empty at render time, and empty routes are filed as soft 404s. Disallow
+    the write and admin surfaces, not the read-only data.
+
+    There is no Content-Signal line and no option to add one. Lighthouse's
+    robots.txt validator does not know the directive, reports it as an
+    unknown directive and takes points off the SEO score of every page on
+    the site. Absence already means "no restriction".
+    """
+    base_url = base_url.rstrip("/")
+    lines: list[str] = ["User-agent: *"]
+    lines += [f"Allow: {p}" for p in allow]
+    lines += [f"Disallow: {p}" for p in disallow]
+
+    for agent in ai_crawlers or ():
+        lines += ["", f"User-agent: {agent}", "Allow: /"]
+
+    lines += list(extra_lines)
+
+    if sitemap:
+        maps = [sitemap] if isinstance(sitemap, str) else list(sitemap)
+        lines.append("")
+        for m in maps:
+            lines.append(f"Sitemap: {m if m.startswith('http') else base_url + m}")
+
+    lines.append("")
+    return "\n".join(lines)

crawlableseo/shell.py

@@ -0,0 +1,134 @@
+"""Inject per-URL metadata and crawlable content into a built SPA shell."""
+
+from __future__ import annotations
+
+import html
+import re
+
+from .head import head_block
+from .page import Page
+from .shell_marker import MARKER
+
+__all__ = ["MARKER", "render_shell"]
+
+_COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL)
+_TITLE_RE = re.compile(r"<title\b[^>]*>.*?</title>", re.IGNORECASE | re.DOTALL)
+_DESC_RE = re.compile(
+    r"""<meta\s+name=["']description["'][^>]*>""", re.IGNORECASE
+)
+
+
+def _comment_spans(template: str) -> list[tuple[int, int]]:
+    return [m.span() for m in _COMMENT_RE.finditer(template)]
+
+
+def _search_outside_comments(
+    pattern: re.Pattern[str], template: str, spans: list[tuple[int, int]]
+) -> re.Match[str] | None:
+    """First match that is not inside an HTML comment.
+
+    A build-time comment that merely mentions ``<title>`` is not the title.
+    Matching it and replacing it destroys the comment's closing marker, which
+    comments out the rest of the head: the stylesheet and the bundle script
+    become comment text, and the site serves a blank page with no console
+    error and no failed request.
+    """
+    pos = 0
+    while pos <= len(template):
+        m = pattern.search(template, pos)
+        if m is None:
+            return None
+        inside = next((b for a, b in spans if a <= m.start() < b), None)
+        if inside is None:
+            return m
+        # Resume after the comment, not after the match: this match started
+        # inside the comment and swallowed the real tag that follows it.
+        pos = inside
+    return None
+
+
+def render_shell(
+    template: str,
+    page: Page,
+    *,
+    canonical: str,
+    base_url: str,
+    site_name: str = "",
+    twitter_site: str = "",
+    default_og_image: str | None = None,
+    mount_id: str = "root",
+) -> str:
+    """Return the shell with this page's tags and crawlable body in place."""
+    if MARKER in template:
+        return template
+
+    page = page.clipped()
+    e = html.escape
+
+    # Both tags are located in one pass over the original document, then the
+    # edits are applied back to front so the earlier offsets stay valid.
+    # Slicing, never re.sub with a replacement string: a title built from a
+    # query parameter can contain a backslash escape, which re would read as
+    # a group reference and raise on, turning an ordinary URL into a 500.
+    spans = _comment_spans(template)
+    title_tag = f"<title>{e(page.title)}</title>"
+    desc_tag = f'<meta name="description" content="{e(page.description)}" />'
+    edits: list[tuple[int, int, str]] = []
+    missing: list[str] = []
+
+    for pattern, tag in ((_TITLE_RE, title_tag), (_DESC_RE, desc_tag)):
+        m = _search_outside_comments(pattern, template, spans)
+        if m:
+            edits.append((m.start(), m.end(), tag))
+        else:
+            missing.append(tag)
+
+    out = template
+    for start, end, tag in sorted(edits, reverse=True):
+        out = out[:start] + tag + out[end:]
+    for tag in missing:
+        out = _insert_into_head(out, tag)
+
+    out = _insert_into_head(
+        out,
+        head_block(
+            page,
+            canonical=canonical,
+            base_url=base_url,
+            site_name=site_name,
+            twitter_site=twitter_site,
+            default_og_image=default_og_image,
+        ),
+    )
+
+    if page.body:
+        out = _fill_mount_node(out, mount_id, page.body)
+    return out
+
+
+def _insert_into_head(template: str, block: str) -> str:
+    idx = template.lower().find("</head>")
+    if idx == -1:
+        return template + "\n" + block
+    return template[:idx] + block + "\n" + template[idx:]
+
+
+_MOUNT_RE_CACHE: dict[str, re.Pattern[str]] = {}
+
+
+def _mount_re(mount_id: str) -> re.Pattern[str]:
+    cached = _MOUNT_RE_CACHE.get(mount_id)
+    if cached is None:
+        cached = re.compile(
+            r"(<div\b[^>]*\bid=[\"']?" + re.escape(mount_id) + r"[\"']?[^>]*>)\s*</div>",
+            re.IGNORECASE,
+        )
+        _MOUNT_RE_CACHE[mount_id] = cached
+    return cached
+
+
+def _fill_mount_node(template: str, mount_id: str, body: str) -> str:
+    m = _mount_re(mount_id).search(template)
+    if not m:
+        return template
+    return template[: m.start()] + m.group(1) + body + "</div>" + template[m.end() :]

crawlableseo/shell_marker.py

@@ -0,0 +1,7 @@
+"""The injection marker, in its own module so head and shell can share it."""
+
+from __future__ import annotations
+
+# Marks a shell that has already been injected, so a second pass is a no-op
+# rather than a document carrying two canonicals.
+MARKER = "<!-- crawlableseo -->"

crawlableseo/site.py

@@ -0,0 +1,270 @@
+"""One declaration of a site; every output is derived from it."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from .llmstxt import LlmsSection, llms_txt
+from .page import NotFound, Page
+from .robots import DEFAULT_AI_CRAWLERS, robots_txt
+from .shell import render_shell
+from .sitemap import SitemapEntry, sitemap_xml
+from .urls import page_url
+
+Resolver = Callable[[str, dict[str, str]], "Page | NotFound | None"]
+UrlSupplier = Callable[[], Iterable["DynamicUrl"]]
+
+
+@dataclass(frozen=True)
+class DynamicUrl:
+    """One URL of a dynamic route, for the sitemap.
+
+    ``params`` must be the same parameters the resolver puts on the page it
+    returns, because both the canonical tag and this entry are built from
+    them by :func:`page_url`.
+    """
+
+    path: str
+    params: dict[str, str] | None = None
+    lastmod: str | None = None
+    changefreq: str = "weekly"
+    priority: float = 0.5
+
+
+@dataclass
+class _Dynamic:
+    prefix: str
+    resolver: Resolver
+    urls: UrlSupplier | None
+
+
+class Site:
+    """The site's crawlable surface.
+
+    A static page is declared once with :meth:`page` and appears in the
+    shell's tags and in the sitemap. A dynamic route is declared with
+    :meth:`dynamic`: a resolver builds the page for a request, and an
+    optional supplier lists that route's URLs for the sitemap.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        *,
+        shell: str | Path | None = None,
+        shell_html: str | None = None,
+        name: str = "",
+        twitter_site: str = "",
+        default_og_image: str | None = None,
+        mount_id: str = "root",
+        default_title: str = "",
+        default_description: str = "",
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.name = name
+        self.twitter_site = twitter_site
+        self.default_og_image = default_og_image
+        self.mount_id = mount_id
+        self.default_title = default_title or name
+        self.default_description = default_description
+        if (shell is None) == (shell_html is None):
+            raise ValueError("pass exactly one of shell (a path) or shell_html (markup)")
+        self._shell_path = Path(shell) if shell is not None else None
+        self._shell_cache: str | None = shell_html
+        self._pages: dict[str, Page] = {}
+        self._dynamic: list[_Dynamic] = []
+        self._noindex_prefixes: list[str] = []
+
+    # -- declaration --------------------------------------------------
+
+    def page(
+        self,
+        path: str,
+        *,
+        title: str,
+        description: str,
+        body: str = "",
+        params: dict[str, str] | None = None,
+        jsonld: tuple[dict[str, object], ...] = (),
+        og_image: str | None = None,
+        index: bool = True,
+        follow: bool = True,
+        in_sitemap: bool = True,
+        changefreq: str = "weekly",
+        priority: float = 0.5,
+        lastmod: str | None = None,
+    ) -> Page:
+        """Declare a static page.
+
+        The arguments are :class:`Page`'s fields, spelled out rather than
+        forwarded as ``**kwargs``: this is the call people write most often,
+        and it should complete and type-check in an editor.
+        """
+        page = Page(
+            title=title,
+            description=description,
+            body=body,
+            params=params or {},
+            jsonld=jsonld,
+            og_image=og_image,
+            index=index,
+            follow=follow,
+            in_sitemap=in_sitemap,
+            changefreq=changefreq,
+            priority=priority,
+            lastmod=lastmod,
+        )
+        self._pages[_normalise(path)] = page
+        return page
+
+    def add(self, path: str, page: Page) -> Page:
+        """Declare a static page from a :class:`Page` you built yourself."""
+        self._pages[_normalise(path)] = page
+        return page
+
+    def dynamic(
+        self, prefix: str, *, urls: UrlSupplier | None = None
+    ) -> Callable[[Resolver], Resolver]:
+        """Register a resolver for every path under ``prefix``.
+
+        The resolver returns a :class:`Page`, or :class:`NotFound` for a URL
+        that has no content. Returning None falls through to the next
+        resolver, then to the site defaults.
+        """
+
+        def decorate(resolver: Resolver) -> Resolver:
+            self._dynamic.append(_Dynamic(_normalise(prefix), resolver, urls))
+            return resolver
+
+        return decorate
+
+    def noindex(self, *prefixes: str) -> None:
+        """Mark private or utility route prefixes as never indexable."""
+        self._noindex_prefixes.extend(_normalise(p) for p in prefixes)
+
+    # -- resolution ---------------------------------------------------
+
+    def resolve(self, path: str, query: dict[str, str] | None = None) -> Page:
+        path = _normalise(path)
+        query = query or {}
+
+        for prefix in self._noindex_prefixes:
+            if path == prefix or path.startswith(prefix.rstrip("/") + "/"):
+                return self._hidden()
+
+        declared = self._pages.get(path)
+        if declared is not None:
+            return declared
+
+        for entry in self._dynamic:
+            if path == entry.prefix or path.startswith(entry.prefix.rstrip("/") + "/"):
+                result = entry.resolver(path, query)
+                if isinstance(result, NotFound):
+                    return self._hidden(status=404)
+                if result is not None:
+                    return result
+
+        # An unknown path is not a second copy of the home page. Answering
+        # 200 there turns every typo and every stale link into an indexable
+        # duplicate of the site's most important page.
+        return self._hidden(status=404)
+
+    def _hidden(self, status: int = 200) -> Page:
+        """The site's fallback card for a page no crawler should index."""
+        return Page(
+            title=self.default_title,
+            description=self.default_description,
+            index=False,
+            follow=False,
+            in_sitemap=False,
+            status=status,
+        )
+
+    def canonical(self, path: str, page: Page) -> str:
+        return page_url(self.base_url, path, page.params)
+
+    # -- outputs ------------------------------------------------------
+
+    def shell(self) -> str:
+        """The built shell, read once and kept in memory.
+
+        Call :meth:`reload_shell` after a new frontend build, or restart the
+        process: a long-lived server otherwise serves the previous bundle's
+        script tags for as long as it runs.
+        """
+        if self._shell_cache is None:
+            assert self._shell_path is not None
+            self._shell_cache = self._shell_path.read_text(encoding="utf-8")
+        return self._shell_cache
+
+    def reload_shell(self) -> None:
+        if self._shell_path is not None:
+            self._shell_cache = None
+
+    def render(self, path: str, query: dict[str, str] | None = None) -> tuple[str, int]:
+        """Return ``(html, status)`` for a request path."""
+        page = self.resolve(path, query)
+        html = render_shell(
+            self.shell(),
+            page,
+            canonical=self.canonical(_normalise(path), page),
+            base_url=self.base_url,
+            site_name=self.name,
+            twitter_site=self.twitter_site,
+            default_og_image=self.default_og_image,
+            mount_id=self.mount_id,
+        )
+        return html, page.status
+
+    def sitemap_entries(self) -> list[SitemapEntry]:
+        entries = [
+            SitemapEntry(
+                loc=page_url(self.base_url, path, page.params),
+                lastmod=page.lastmod,
+                changefreq=page.changefreq,
+                priority=page.priority,
+            )
+            for path, page in sorted(self._pages.items())
+            if page.in_sitemap and page.index
+        ]
+        for entry in self._dynamic:
+            for url in entry.urls() if entry.urls else ():
+                entries.append(
+                    SitemapEntry(
+                        loc=page_url(self.base_url, url.path, url.params),
+                        lastmod=url.lastmod,
+                        changefreq=url.changefreq,
+                        priority=url.priority,
+                    )
+                )
+        return entries
+
+    def sitemap(self) -> str:
+        return sitemap_xml(self.sitemap_entries())
+
+    def robots(
+        self,
+        *,
+        disallow: Sequence[str] = (),
+        ai_crawlers: Sequence[str] | None = DEFAULT_AI_CRAWLERS,
+        extra_lines: Sequence[str] = (),
+    ) -> str:
+        return robots_txt(
+            self.base_url,
+            disallow=list(disallow) + self._noindex_prefixes,
+            ai_crawlers=ai_crawlers,
+            extra_lines=extra_lines,
+        )
+
+    def llms(
+        self, summary: str, sections: Iterable[LlmsSection], *, notes: Sequence[str] = ()
+    ) -> str:
+        return llms_txt(self.name, summary, sections, notes=notes)
+
+
+def _normalise(path: str) -> str:
+    if not path.startswith("/"):
+        path = "/" + path
+    return path.rstrip("/") or "/"

crawlableseo/sitemap.py

@@ -0,0 +1,45 @@
+"""sitemap.xml, built from the same URL function as the canonical tag."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from .urls import xml_escape
+
+# The protocol's own ceiling. Past it a sitemap must be split and listed in
+# an index; this library raises rather than shipping a file crawlers reject.
+MAX_URLS = 50_000
+
+
+@dataclass(frozen=True)
+class SitemapEntry:
+    loc: str
+    lastmod: str | None = None
+    changefreq: str = "weekly"
+    priority: float = 0.5
+
+
+def sitemap_xml(entries: Iterable[SitemapEntry]) -> str:
+    entries = list(entries)
+    if len(entries) > MAX_URLS:
+        raise ValueError(
+            f"a sitemap holds at most {MAX_URLS} URLs, got {len(entries)}; "
+            "split it and publish a sitemap index"
+        )
+    body = "\n".join(_url_tag(e) for e in entries)
+    return (
+        '<?xml version="1.0" encoding="UTF-8"?>\n'
+        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
+        f"{body}\n"
+        "</urlset>\n"
+    )
+
+
+def _url_tag(entry: SitemapEntry) -> str:
+    parts = [f"  <url><loc>{xml_escape(entry.loc)}</loc>"]
+    if entry.lastmod:
+        parts.append(f"<lastmod>{xml_escape(entry.lastmod)}</lastmod>")
+    parts.append(f"<changefreq>{entry.changefreq}</changefreq>")
+    parts.append(f"<priority>{entry.priority:.1f}</priority></url>")
+    return "".join(parts)

crawlableseo/urls.py

@@ -0,0 +1,36 @@
+"""The single place a page's URL is built.
+
+Every URL this library emits - the canonical tag, the sitemap entry, the
+IndexNow submission, the llms.txt link - comes from :func:`page_url`. When a
+canonical and its sitemap entry are built by two different pieces of code
+they drift, usually over one query parameter, and a search engine treats the
+two spellings as two pages with identical content.
+"""
+
+from __future__ import annotations
+
+from urllib.parse import quote
+
+
+def page_url(base_url: str, path: str, params: dict[str, str] | None = None) -> str:
+    base_url = base_url.rstrip("/")
+    if not path.startswith("/"):
+        path = "/" + path
+    if path != "/":
+        path = path.rstrip("/") or "/"
+    url = base_url + quote(path, safe="/-._~")
+    if params:
+        query = "&".join(
+            f"{quote(k, safe='')}={quote(v, safe='')}" for k, v in sorted(params.items())
+        )
+        url = f"{url}?{query}"
+    return url
+
+
+def xml_escape(value: str) -> str:
+    return (
+        value.replace("&", "&")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace('"', "&quot;")
+    )

crawlableseo-0.1.0.dist-info/METADATA

@@ -0,0 +1,208 @@
+Metadata-Version: 2.5
+Name: crawlableseo
+Version: 0.1.0
+Summary: Make a client-rendered SPA crawlable from its own HTML shell: meta tags, JSON-LD, robots.txt, sitemap.xml and llms.txt from one declaration.
+Project-URL: Homepage, https://github.com/kulykivska/crawlableseo
+Project-URL: Issues, https://github.com/kulykivska/crawlableseo/issues
+Project-URL: Changelog, https://github.com/kulykivska/crawlableseo/blob/main/CHANGELOG.md
+Author: Yuliia Kulykivska
+License: MIT License
+        
+        Copyright (c) 2026 Yuliia Kulykivska
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: crawler,fastapi,indexnow,json-ld,llms-txt,robots,seo,sitemap,spa,starlette
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: fastapi>=0.100; extra == 'dev'
+Requires-Dist: httpx>=0.24; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.100; extra == 'fastapi'
+Provides-Extra: indexnow
+Requires-Dist: httpx>=0.24; extra == 'indexnow'
+Description-Content-Type: text/markdown
+
+# crawlableseo
+
+Make a client-rendered single-page app crawlable from its own HTML shell.
+
+No headless browser, no SSR framework, no third-party prerendering service. Your
+Python server fills in the shell's `<head>` and mount node for each URL, and the
+same declaration produces `robots.txt`, `sitemap.xml` and `llms.txt`.
+
+```bash
+pip install crawlableseo
+```
+
+## The problem
+
+A Vite or CRA build ships one `index.html` with an empty `<div id="root">`. Every
+URL on the site returns the same document: the same title, the same description,
+no content. Google renders JavaScript and will often cope; Bing is slower to; and
+the crawlers behind AI answers — GPTBot, PerplexityBot, ClaudeBot — mostly read
+the HTML they are given. What they are given is an empty div.
+
+The usual answers are to adopt a meta-framework, run a headless browser per
+request, or pay a prerendering service. This library takes the fourth option:
+serve the same static shell, but write the real title, description, canonical,
+Open Graph, JSON-LD and a block of readable HTML into it before it goes out.
+
+## Quickstart
+
+```python
+from fastapi import FastAPI
+from crawlableseo import DynamicUrl, NotFound, Page, Site
+from crawlableseo.integrations.fastapi import router
+
+site = Site(
+    "https://example.com",
+    shell="frontend/dist/index.html",
+    name="Example",
+    default_title="Example",
+    default_description="What this site is, in one sentence.",
+)
+
+site.page(
+    "/pricing",
+    title="Pricing | Example",
+    description="Three plans, what each one includes, and what they cost.",
+    body="<h1>Pricing</h1><p>Free, Pro and Team...</p>",
+    priority=0.8,
+)
+
+site.noindex("/admin", "/login", "/settings")
+
+
+@site.dynamic("/product", urls=lambda: [DynamicUrl("/product", {"id": p.id}) for p in catalogue()])
+def product(path: str, query: dict[str, str]) -> Page | NotFound | None:
+    item = lookup(query.get("id"))
+    if item is None:
+        return NotFound()
+    return Page(
+        title=f"{item.name} | Example",
+        description=item.summary,
+        params={"id": item.id},
+        body=f"<h1>{item.name}</h1><p>{item.summary}</p>",
+        jsonld=({"@context": "https://schema.org", "@type": "Product", "name": item.name},),
+    )
+
+
+app = FastAPI()
+# ... your API routes ...
+app.include_router(router(site, disallow=["/api/admin"]))  # mount last
+```
+
+That serves `/robots.txt`, `/sitemap.xml` and the SPA catch-all. Add `llms=` for
+`/llms.txt` and `indexnow=` to publish an IndexNow key file.
+
+Not using FastAPI? Everything underneath is a plain function:
+
+```python
+from crawlableseo import Page, render_shell, robots_txt, sitemap_xml
+```
+
+## What it does
+
+- **Per-URL `<head>`** — title, description, canonical, robots, Open Graph,
+  Twitter cards, and any number of JSON-LD objects.
+- **Crawlable content** — HTML written into the mount node. Your framework
+  replaces the node's children when it mounts, so users never see it; a crawler
+  that does not run JavaScript reads it as the page.
+- **`robots.txt`, `sitemap.xml`, `llms.txt`** — from the same declaration, so a
+  page cannot be in one and missing from another.
+- **IndexNow** — submit changed URLs to Bing (and through it, ChatGPT search)
+  instead of waiting for the next crawl.
+- **Honest status codes** — a URL with no content answers 404, not a 200 with an
+  empty screen.
+
+## What it does not do
+
+- It never writes your copy. You supply the text; the library places it.
+- It does not render your JavaScript. If a page's content exists only after a
+  client-side fetch, give the resolver access to the same data on the server.
+- It is not a meta-framework and will not become one.
+
+## Bugs this prevents
+
+Each of these was shipped to production on a live site before it was understood.
+They are the reason the library exists, and every one has a test.
+
+**A `<title>` mentioned in a build comment is not the title.** A shell carried a
+comment explaining that the server rewrites `<title>`. A naive search-and-replace
+matched that mention; the replacement ate the comment's closing `-->`, which
+commented out the rest of `<head>` — stylesheet and bundle script included. The
+site served a blank page with no console error and no failed request.
+
+**Canonical and sitemap drift.** When the canonical tag and the sitemap entry are
+built by two pieces of code, they disagree over a query parameter sooner or later,
+and the two spellings become two pages with identical content. Here both come from
+one function, and there is a test that fails if they ever differ.
+
+**Blocking your own API starves the renderer.** `Disallow: /api/` in `robots.txt`
+looks tidy. A crawler renders the page like a browser, so blocking the JSON the
+app fetches leaves every route empty at render time. On one site that produced 22
+soft 404s and 225 URLs stuck at "Discovered – currently not indexed". Disallow the
+write and admin surfaces; leave the read-only data alone.
+
+**`Content-Signal` costs more than it gives.** Lighthouse's `robots.txt` validator
+does not know the directive, reports it as unknown, and takes points off the SEO
+score of every page. Its absence already means no restriction, so this library has
+no option to emit it.
+
+**A `</` in your data closes the script tag.** One product name with a slash in it
+and the JSON-LD block ends early, taking the rest of the document with it. Escaped
+here, once.
+
+**A backslash in a title raises a 500.** Titles built from query parameters can
+contain anything; `re.sub` reads `\1` in a replacement string as a group
+reference. An ordinary crafted URL becomes a server error.
+
+**A 200 on an unknown path is a duplicate home page.** Every typo and stale link
+becomes an indexable copy of your most important page. Unknown paths answer 404.
+
+**`noindex` is not `nofollow`.** A page can be worth keeping out of the index while
+its outgoing links are still worth crawling. They are separate flags.
+
+**Do not detach the crawlable block early.** If you remove it before your framework
+mounts, the page paints, empties, then repaints: on one site that measured a
+cumulative layout shift of 0.28. Let the framework replace it.
+
+## Compatibility
+
+Python 3.10+. No required dependencies. `crawlableseo[indexnow]` adds `httpx`;
+`crawlableseo[fastapi]` adds FastAPI for the router.
+
+## License
+
+MIT

crawlableseo-0.1.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+crawlableseo/__init__.py,sha256=FFUM8__7X8zs3swBAUOQ5rKNwalZCHkD_2NdZPYamYA,1082
+crawlableseo/head.py,sha256=2BOodVIwYq3BBMwV3V6AhFW4OxSkeQl6t2rRozaSM4w,2169
+crawlableseo/indexnow.py,sha256=R0gRG-EOysDymV1MLwcx6r-VZM7tzdADEFTIk13RRLk,4008
+crawlableseo/llmstxt.py,sha256=loamGNHLVlR68fx9MKTsyPgwPJxd51MvWpG7KGQbCso,1134
+crawlableseo/page.py,sha256=g8B91Dbzo9B_PNdcWbwl_O6cM6BQzHjZiQwIgzsxxIY,2795
+crawlableseo/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlableseo/robots.py,sha256=pcmQhTENGrG6dLcehWJo54hTEP1TZlDyfkGP1e4-irw,1980
+crawlableseo/shell.py,sha256=iWU5iOIV4vu6J-5yXZ90e7N8j1faBRTKFG8HRNVl3Ec,4250
+crawlableseo/shell_marker.py,sha256=Jg7NdSRGHmMVfDjN-YK0oiEjGuIXfciog15VlVNJvBY,274
+crawlableseo/site.py,sha256=Jzzwv0SDEGqr7121DwwcrY1xLPsbo86zP4c4a5MpBR8,9164
+crawlableseo/sitemap.py,sha256=bA4GS_xdOl5hi2slsYf1i9wcnj69hVxT9-7fO3dERNI,1407
+crawlableseo/urls.py,sha256=vj4aqQiafxtImw_UHh994QYo3m9bs-cJtJIimLLFpV8,1127
+crawlableseo/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlableseo/integrations/fastapi.py,sha256=i9M3ablddBMivtzcxDGHpemgAQkpficWD4n2YACo-ZA,3047
+crawlableseo-0.1.0.dist-info/METADATA,sha256=fpxF4_zc1A3Kn6FNtwHy_LDRd50I0e3wH513G3Ib4Xk,9082
+crawlableseo-0.1.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
+crawlableseo-0.1.0.dist-info/licenses/LICENSE,sha256=Q8_f8lqxmXMni_cH9slkOwVhwcaztOi8BIwK6nSEC9k,1074
+crawlableseo-0.1.0.dist-info/RECORD,,

crawlableseo-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.3
+Root-Is-Purelib: true
+Tag: py3-none-any

crawlableseo-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Yuliia Kulykivska
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.