sphinx-llm-friendly 0.1.0__tar.gz

sphinx_llm_friendly-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.egg-info/
+.tox/
+build/
+dist/
+tests/source/library/

sphinx_llm_friendly-0.1.0/LICENSE

@@ -0,0 +1,23 @@
+The MIT License (MIT)
+
+Copyright (c) Scrapy developers.
+Copyright (c) 2023-2026, Liran Funaro.
+Copyright (c) 2025 Jared Dillard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sphinx_llm_friendly-0.1.0/PKG-INFO

@@ -0,0 +1,100 @@
+Metadata-Version: 2.5
+Name: sphinx-llm-friendly
+Version: 0.1.0
+Summary: Sphinx extension to make documentation LLM-friendly
+Project-URL: Source, https://github.com/scrapy/sphinx-llm-friendly
+Project-URL: Tracker, https://github.com/scrapy/sphinx-llm-friendly/issues
+Project-URL: Release notes, https://github.com/scrapy/sphinx-llm-friendly/blob/main/CHANGES.rst
+Author-email: Scrapy developers <opensource@zyte.com>
+Maintainer-email: Scrapy developers <opensource@zyte.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Sphinx :: Extension
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Topic :: Documentation :: Sphinx
+Requires-Python: >=3.10
+Requires-Dist: docutils
+Requires-Dist: sphinx>=7.4
+Requires-Dist: tabulate
+Description-Content-Type: text/x-rst
+
+===================
+sphinx-llm-friendly
+===================
+
+`Sphinx <https://www.sphinx-doc.org/>`_ extension that makes documentation
+LLM-friendly:
+
+-   A Markdown version of every page, next to its HTML version.
+
+-   `llms.txt <https://llmstxt.org/>`_, with a link to the Markdown version of
+    every page.
+
+-   ``llms-full.txt``, with the Markdown version of all pages.
+
+-   HTML pages point to their Markdown version with a ``<link
+    rel="alternate" type="text/markdown">`` tag, and get a button to copy
+    their Markdown version.
+
+Setup
+=====
+
+#.  Install it:
+
+    .. code-block:: shell
+
+        pip install sphinx-llm-friendly
+
+#.  Add it to ``extensions`` in ``conf.py``:
+
+    .. code-block:: python
+
+        extensions = [
+            # …
+            "sphinx_llm_friendly",
+        ]
+
+#.  Build your documentation:
+
+    .. code-block:: shell
+
+        sphinx-llm-friendly build docs docs/_build
+
+    The output is in ``docs/_build/all``.
+
+    It runs the ``html``, ``llm_markdown`` and ``llm_singlemarkdown`` builders
+    in parallel, into subdirectories of ``docs/_build``, and merges their
+    output. In the Markdown output, links to sites from ``intersphinx_mapping``
+    that serve Markdown point to the Markdown version of their pages.
+
+    From Python, call ``sphinx_llm_friendly.build(source_dir, build_dir)``,
+    which returns the output directory.
+
+Configuration
+=============
+
+``llm_friendly_exclude``
+    List of patterns, with the syntax of ``exclude_patterns``, of documents to
+    leave out of the Markdown output and ``llms.txt``. Default: ``[]``.
+
+``llm_friendly_llms_txt_summary``
+    Summary for ``llms.txt``. Default: the first paragraph of the root
+    document.
+
+``llm_friendly_llms_txt_toctree_only``
+    If ``True``, ``llms.txt`` only lists documents reachable through toctrees
+    from the root document. Default: ``False``.
+
+Nodes from third-party extensions need Markdown handlers, registered with
+``app.add_node()`` for the ``llm_markdown`` and ``llm_singlemarkdown``
+builders.

sphinx_llm_friendly-0.1.0/README.rst

@@ -0,0 +1,70 @@
+===================
+sphinx-llm-friendly
+===================
+
+`Sphinx <https://www.sphinx-doc.org/>`_ extension that makes documentation
+LLM-friendly:
+
+-   A Markdown version of every page, next to its HTML version.
+
+-   `llms.txt <https://llmstxt.org/>`_, with a link to the Markdown version of
+    every page.
+
+-   ``llms-full.txt``, with the Markdown version of all pages.
+
+-   HTML pages point to their Markdown version with a ``<link
+    rel="alternate" type="text/markdown">`` tag, and get a button to copy
+    their Markdown version.
+
+Setup
+=====
+
+#.  Install it:
+
+    .. code-block:: shell
+
+        pip install sphinx-llm-friendly
+
+#.  Add it to ``extensions`` in ``conf.py``:
+
+    .. code-block:: python
+
+        extensions = [
+            # …
+            "sphinx_llm_friendly",
+        ]
+
+#.  Build your documentation:
+
+    .. code-block:: shell
+
+        sphinx-llm-friendly build docs docs/_build
+
+    The output is in ``docs/_build/all``.
+
+    It runs the ``html``, ``llm_markdown`` and ``llm_singlemarkdown`` builders
+    in parallel, into subdirectories of ``docs/_build``, and merges their
+    output. In the Markdown output, links to sites from ``intersphinx_mapping``
+    that serve Markdown point to the Markdown version of their pages.
+
+    From Python, call ``sphinx_llm_friendly.build(source_dir, build_dir)``,
+    which returns the output directory.
+
+Configuration
+=============
+
+``llm_friendly_exclude``
+    List of patterns, with the syntax of ``exclude_patterns``, of documents to
+    leave out of the Markdown output and ``llms.txt``. Default: ``[]``.
+
+``llm_friendly_llms_txt_summary``
+    Summary for ``llms.txt``. Default: the first paragraph of the root
+    document.
+
+``llm_friendly_llms_txt_toctree_only``
+    If ``True``, ``llms.txt`` only lists documents reachable through toctrees
+    from the root document. Default: ``False``.
+
+Nodes from third-party extensions need Markdown handlers, registered with
+``app.add_node()`` for the ``llm_markdown`` and ``llm_singlemarkdown``
+builders.

sphinx_llm_friendly-0.1.0/pyproject.toml

@@ -0,0 +1,141 @@
+[build-system]
+requires = ["hatchling>=1.27.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "sphinx-llm-friendly"
+version = "0.1.0"
+description = "Sphinx extension to make documentation LLM-friendly"
+dependencies = [
+    "docutils",
+    "sphinx>=7.4",
+    "tabulate",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Framework :: Sphinx :: Extension",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3.15",
+    "Topic :: Documentation :: Sphinx",
+]
+license = "MIT"
+license-files = ["LICENSE"]
+readme = "README.rst"
+requires-python = ">=3.10"
+authors = [{ name = "Scrapy developers", email = "opensource@zyte.com" }]
+maintainers = [{ name = "Scrapy developers", email = "opensource@zyte.com" }]
+
+[project.scripts]
+sphinx-llm-friendly = "sphinx_llm_friendly._build:main"
+
+[project.entry-points."sphinx.builders"]
+llm_markdown = "sphinx_llm_friendly"
+llm_singlemarkdown = "sphinx_llm_friendly"
+
+[project.urls]
+Source = "https://github.com/scrapy/sphinx-llm-friendly"
+Tracker = "https://github.com/scrapy/sphinx-llm-friendly/issues"
+"Release notes" = "https://github.com/scrapy/sphinx-llm-friendly/blob/main/CHANGES.rst"
+
+[tool.hatch.build.targets.sdist]
+include = ["/sphinx_llm_friendly", "/tests"]
+
+[tool.mypy]
+strict = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.bumpversion]
+current_version = "0.1.0"
+commit = true
+tag = true
+tag_name = "{new_version}"
+
+[[tool.bumpversion.files]]
+filename = "CHANGES.rst"
+search = "\\(unreleased\\)$"
+replace = "({now:%Y-%m-%d})"
+regex = true
+
+[tool.ruff.lint]
+extend-select = [
+    # flake8-bugbear
+    "B",
+    # flake8-comprehensions
+    "C4",
+    # pydocstyle
+    "D",
+    # flake8-future-annotations
+    "FA",
+    # flynt
+    "FLY",
+    # refurb
+    "FURB",
+    # isort
+    "I",
+    # flake8-implicit-str-concat
+    "ISC",
+    # flake8-logging
+    "LOG",
+    # Perflint
+    "PERF",
+    # pygrep-hooks
+    "PGH",
+    # flake8-pie
+    "PIE",
+    # pylint
+    "PL",
+    # flake8-pytest-style
+    "PT",
+    # flake8-use-pathlib
+    "PTH",
+    # flake8-pyi
+    "PYI",
+    # flake8-quotes
+    "Q",
+    # flake8-return
+    "RET",
+    # flake8-raise
+    "RSE",
+    # Ruff-specific rules
+    "RUF",
+    # flake8-bandit
+    "S",
+    # flake8-simplify
+    "SIM",
+    # flake8-slots
+    "SLOT",
+    # flake8-debugger
+    "T10",
+    # flake8-type-checking
+    "TC",
+    # flake8-tidy-imports
+    "TID",
+    # pyupgrade
+    "UP",
+    # pycodestyle warnings
+    "W",
+    # flake8-2020
+    "YTT",
+]
+ignore = [
+    # No public API to document
+    "D",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"sphinx_llm_friendly/_contexts.py" = ["B008", "PT018", "S101"]
+"sphinx_llm_friendly/_translator.py" = ["PLR0911", "S101"]
+"tests/*" = ["S101"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "pep257"

sphinx_llm_friendly-0.1.0/sphinx_llm_friendly/__init__.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import posixpath
+from importlib.metadata import version
+from typing import TYPE_CHECKING, Any
+
+from sphinx.util.osutil import relative_uri
+
+from ._build import build, remember_intersphinx_urls
+from ._builder import MarkdownBuilder
+from ._exclude import is_excluded
+from ._llms_txt import write_llms_txt
+from ._singlemarkdown import SingleFileMarkdownBuilder
+
+if TYPE_CHECKING:
+    from docutils import nodes
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
+
+__all__ = ["build", "setup"]
+
+_COPY_AS_MARKDOWN_BUTTON_JS = """
+(function () {
+    var DEFAULT_LABEL = 'M\u2193';
+    var SUCCESS_LABEL = 'Copied';
+    var ERROR_LABEL = 'Error';
+
+    function markdownPathFromCurrentPage(pathname) {
+        if (pathname.endsWith('.html')) {
+            return pathname.slice(0, -5) + '.md';
+        }
+        if (pathname.endsWith('/')) {
+            return pathname + 'index.md';
+        }
+        var lastPart = pathname.split('/').pop() || '';
+        if (!lastPart.includes('.')) {
+            return pathname + '.md';
+        }
+        return pathname + '.md';
+    }
+
+    async function copyToClipboard(text) {
+        if (navigator.clipboard && window.isSecureContext) {
+            await navigator.clipboard.writeText(text);
+            return;
+        }
+        var textarea = document.createElement('textarea');
+        textarea.value = text;
+        textarea.setAttribute('readonly', 'readonly');
+        textarea.style.position = 'fixed';
+        textarea.style.left = '-9999px';
+        document.body.appendChild(textarea);
+        textarea.select();
+        document.execCommand('copy');
+        document.body.removeChild(textarea);
+    }
+
+    function setTemporaryLabel(button, label, previousLabel) {
+        var prev = typeof previousLabel !== 'undefined' ? previousLabel : button.textContent;
+        button.textContent = label;
+        window.setTimeout(function () {
+            button.textContent = prev;
+            button.disabled = false;
+        }, 1000);
+    }
+
+    async function onButtonClick(button) {
+        var previousLabel = button.textContent;
+        button.disabled = true;
+        button.textContent = '...';
+        try {
+            var mdPath = markdownPathFromCurrentPage(window.location.pathname);
+            var response = await fetch(mdPath, { credentials: 'same-origin' });
+            if (!response.ok) {
+                throw new Error('Unable to fetch markdown source');
+            }
+            var markdown = await response.text();
+            await copyToClipboard(markdown);
+            setTemporaryLabel(button, SUCCESS_LABEL, previousLabel);
+        } catch (_error) {
+            setTemporaryLabel(button, ERROR_LABEL, previousLabel);
+        }
+    }
+
+    function addStyles() {
+        var style = document.createElement('style');
+        style.textContent = [
+            '.scrapy-copy-as-markdown {',
+            '  display: inline-block;',
+            '  margin-left: 0.25rem;',
+            '  border: 1px solid #c9d4de;',
+            '  border-radius: 0.45rem;',
+            '  background: #ffffff;',
+            '  color: #233a50;',
+            '  font: inherit;',
+            '  font-size: 0.875rem;',
+            '  line-height: 1;',
+            '  padding: 0.25rem 0.25rem;',
+            '  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.08);',
+            '  cursor: pointer;',
+            '}',
+            '.scrapy-copy-as-markdown:hover {',
+            '  background: #f3f7fb;',
+            '}',
+            '.scrapy-copy-as-markdown:disabled {',
+            '  opacity: 0.75;',
+            '  cursor: default;',
+            '}',
+            '.scrapy-copy-as-markdown-title-wrapper {',
+            '  display: flex;',
+            '  align-items: center;',
+            '  justify-content: space-between;',
+            '  gap: 1rem;',
+            '  width: 100%;',
+            '}',
+            '.scrapy-copy-as-markdown-title-wrapper h1 {',
+            '  margin: 0;',
+            '}',
+        ].join('\\n');
+        document.head.appendChild(style);
+    }
+
+    function addButton() {
+        if (!document.body || document.querySelector('.scrapy-copy-as-markdown')) {
+            return;
+        }
+
+        addStyles();
+        var button = document.createElement('button');
+        button.type = 'button';
+        button.className = 'scrapy-copy-as-markdown';
+        button.title = 'Copy this page as Markdown';
+        button.setAttribute('aria-label', 'Copy this page as Markdown');
+        button.textContent = DEFAULT_LABEL;
+        button.addEventListener('click', function () {
+            onButtonClick(button);
+        });
+
+        var h1 = document.querySelector('#main h1') || document.querySelector('h1');
+        if (h1 && h1.parentNode) {
+            var parent = h1.parentNode;
+            var wrapper = document.createElement('div');
+            wrapper.className = 'scrapy-copy-as-markdown-title-wrapper';
+            parent.replaceChild(wrapper, h1);
+            wrapper.appendChild(h1);
+            wrapper.appendChild(button);
+        } else {
+            // fallback: insert at top of body but keep within the first container
+            var container = document.body.firstElementChild || document.body;
+            container.insertBefore(button, container.firstChild);
+        }
+    }
+
+    if (document.readyState === 'loading') {
+        document.addEventListener('DOMContentLoaded', addButton);
+    } else {
+        addButton();
+    }
+})();
+"""
+
+
+def _on_builder_inited(app: Sphinx) -> None:
+    if isinstance(app.builder, MarkdownBuilder):
+        app.config.exclude_patterns = [
+            *app.config.exclude_patterns,
+            *app.config.llm_friendly_exclude,
+        ]
+    elif app.builder.format == "html":
+        app.add_js_file(None, body=_COPY_AS_MARKDOWN_BUTTON_JS)
+
+
+def _add_markdown_alternate_link(
+    app: Sphinx,
+    pagename: str,
+    templatename: str,
+    context: dict[str, Any],
+    doctree: nodes.document | None,
+) -> None:
+    if (
+        app.builder.format != "html"
+        or pagename not in app.env.found_docs
+        or is_excluded(app, pagename)
+    ):
+        return
+    html_uri = app.builder.get_target_uri(pagename)
+    md_uri = f"{posixpath.splitext(html_uri)[0]}.md"
+    href = relative_uri(html_uri, md_uri)
+    context["metatags"] = context.get("metatags", "") + (
+        f'\n<link rel="alternate" type="text/markdown" href="{href}">'
+    )
+
+
+def _on_build_finished(app: Sphinx, exception: Exception | None) -> None:
+    if exception is None and app.builder.name in {"html", "dirhtml"}:
+        write_llms_txt(app)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    app.add_builder(MarkdownBuilder)
+    app.add_builder(SingleFileMarkdownBuilder)
+
+    app.add_config_value("llm_friendly_exclude", [], "env", types=frozenset({list}))
+    app.add_config_value(
+        "llm_friendly_llms_txt_summary",
+        None,
+        "html",
+        types=frozenset({str, type(None)}),
+    )
+    app.add_config_value(
+        "llm_friendly_llms_txt_toctree_only", False, "html", types=frozenset({bool})
+    )
+
+    app.connect("builder-inited", _on_builder_inited)
+    app.connect("html-page-context", _add_markdown_alternate_link)
+    app.connect("build-finished", _on_build_finished)
+    app.connect("build-finished", remember_intersphinx_urls)
+
+    return {
+        "version": version("sphinx-llm-friendly"),
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

sphinx_llm_friendly-0.1.0/sphinx_llm_friendly/_build.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from sphinx.cmd.build import main as sphinx_build_main
+
+from ._intersphinx import rewrite_intersphinx_links
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+_BUILDERS = ("html", "llm_markdown", "llm_singlemarkdown")
+
+_intersphinx_urls: set[str] = set()
+
+
+def remember_intersphinx_urls(app: Sphinx, exception: Exception | None) -> None:
+    """Store the site URLs of ``intersphinx_mapping``, for :func:`_run_builder`
+    to report them to the parent process.
+    """
+    mapping = getattr(app.config, "intersphinx_mapping", {})
+    _intersphinx_urls.clear()
+    _intersphinx_urls.update(uri for _name, (uri, _inventories) in mapping.values())
+
+
+def _run_builder(builder: str, source_dir: Path, build_dir: Path) -> set[str]:
+    exit_code = sphinx_build_main(
+        ["-b", builder, str(source_dir), str(build_dir / builder)]
+    )
+    if exit_code != 0:
+        msg = f"sphinx builder failed for '{builder}' with exit code {exit_code}"
+        raise RuntimeError(msg)
+    return set(_intersphinx_urls)
+
+
+def build(source_dir: Path, build_dir: Path) -> Path:
+    """Build the documentation in *source_dir* as HTML and Markdown, and
+    return the directory inside *build_dir* where the outputs are merged.
+    """
+    build_dir.mkdir(parents=True, exist_ok=True)
+    intersphinx_urls: set[str] = set()
+    with concurrent.futures.ProcessPoolExecutor(max_workers=len(_BUILDERS)) as pool:
+        futures = [
+            pool.submit(_run_builder, builder, source_dir, build_dir)
+            for builder in _BUILDERS
+        ]
+        for future in concurrent.futures.as_completed(futures):
+            intersphinx_urls |= future.result()
+
+    output_dir = build_dir / "all"
+    output_dir.mkdir(parents=True, exist_ok=True)
+    shutil.copytree(build_dir / "html", output_dir, dirs_exist_ok=True)
+    shutil.copytree(build_dir / "llm_markdown", output_dir, dirs_exist_ok=True)
+    shutil.copy2(
+        build_dir / "llm_singlemarkdown" / "index.md", output_dir / "llms-full.txt"
+    )
+    rewrite_intersphinx_links(
+        [
+            *sorted(output_dir.rglob("*.md")),
+            output_dir / "llms.txt",
+            output_dir / "llms-full.txt",
+        ],
+        intersphinx_urls,
+    )
+    return output_dir
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="sphinx-llm-friendly")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    build_parser = subparsers.add_parser(
+        "build", help="build HTML and Markdown documentation"
+    )
+    build_parser.add_argument("source_dir", type=Path)
+    build_parser.add_argument("build_dir", type=Path)
+    args = parser.parse_args(argv)
+    try:
+        output_dir = build(args.source_dir, args.build_dir)
+    except RuntimeError as error:
+        parser.exit(1, f"{error}\n")
+    print(f"\nDocumentation generated in {output_dir}.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())