sphinx-gp-llms 0.0.1a24__tar.gz

sphinx_gp_llms-0.0.1a24/.gitignore

@@ -0,0 +1,233 @@
+# Node
+node_modules/
+*.tsbuildinfo
+.vitest-cache/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Generated by sphinx_fonts extension (downloaded at build time)
+docs/_static/fonts/
+docs/_static/css/fonts.css
+
+# Claude Code
+**/CLAUDE.local.md
+**/CLAUDE.*.md
+**/.claude/settings.local.json
+
+# Playwright MCP
+.playwright-mcp/
+
+# Repo-local pytest mirror (do not track — validator-only)
+out/
+
+# Misc
+.vim/
+*.lprof
+pip-wheel-metadata/
+monkeytype.sqlite3

sphinx_gp_llms-0.0.1a24/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: sphinx-gp-llms
+Version: 0.0.1a24
+Summary: LLM-friendly documentation outputs for Sphinx — llms.txt, llms-full.txt, docs.json, per-page Markdown
+Project-URL: Repository, https://github.com/git-pull/gp-sphinx
+Author-email: Tony Narlock <tony@git-pull.com>
+License: MIT
+Keywords: ai,documentation,llm,llms-txt,sphinx
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Sphinx
+Classifier: Framework :: Sphinx :: Extension
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Documentation
+Classifier: Topic :: Documentation :: Sphinx
+Classifier: Typing :: Typed
+Requires-Python: <4.0,>=3.10
+Requires-Dist: sphinx>=8.1
+Description-Content-Type: text/markdown
+
+# sphinx-gp-llms
+
+LLM-friendly documentation outputs for Sphinx.
+
+Generates `llms.txt`, `llms-full.txt`, `docs.json`, and per-page `.md`
+twin files during the standard HTML build.

sphinx_gp_llms-0.0.1a24/README.md

@@ -0,0 +1,6 @@
+# sphinx-gp-llms
+
+LLM-friendly documentation outputs for Sphinx.
+
+Generates `llms.txt`, `llms-full.txt`, `docs.json`, and per-page `.md`
+twin files during the standard HTML build.

sphinx_gp_llms-0.0.1a24/pyproject.toml

@@ -0,0 +1,43 @@
+[project]
+name = "sphinx-gp-llms"
+version = "0.0.1a24"
+description = "LLM-friendly documentation outputs for Sphinx — llms.txt, llms-full.txt, docs.json, per-page Markdown"
+requires-python = ">=3.10,<4.0"
+authors = [
+  {name = "Tony Narlock", email = "tony@git-pull.com"}
+]
+license = { text = "MIT" }
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "License :: OSI Approved :: MIT License",
+  "Framework :: Sphinx",
+  "Framework :: Sphinx :: Extension",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Topic :: Documentation",
+  "Topic :: Documentation :: Sphinx",
+  "Typing :: Typed",
+]
+readme = "README.md"
+keywords = ["sphinx", "llm", "documentation", "ai", "llms-txt"]
+dependencies = [
+  "sphinx>=8.1",
+]
+
+[project.urls]
+Repository = "https://github.com/git-pull/gp-sphinx"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/sphinx_gp_llms"]
+
+[tool.gp-sphinx.docs]
+showcase = ["dependents"]

sphinx_gp_llms-0.0.1a24/src/sphinx_gp_llms/__init__.py

@@ -0,0 +1,218 @@
+"""LLM-friendly documentation outputs for Sphinx.
+
+Generates ``llms.txt``, ``llms-full.txt``, ``docs.json``, and per-page
+``.md`` twin files during the standard HTML build, following conventions
+established by llmstxt.org (Jeremy Howard / Answer.AI), Cloudflare
+("Markdown for Agents"), Mintlify, and Lakebed (Ping).
+
+The extension hooks into ``build-finished`` to write output files and
+``html-page-context`` to inject footer link variables into the template
+context.
+
+Examples
+--------
+>>> from sphinx_gp_llms import setup
+>>> callable(setup)
+True
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import typing as t
+
+from sphinx.errors import ExtensionError
+from sphinx.util.logging import getLogger
+
+if t.TYPE_CHECKING:
+    from docutils import nodes
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
+
+_EXTENSION_VERSION = "0.0.1a24"
+
+logger = getLogger(__name__)
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+__all__ = ["setup"]
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    """Register config values and connect build hooks.
+
+    Parameters
+    ----------
+    app : Sphinx
+        Sphinx application instance.
+
+    Returns
+    -------
+    ExtensionMetadata
+        Extension metadata with version and parallel-build flags.
+
+    Examples
+    --------
+    >>> from sphinx_gp_llms import setup
+    >>> callable(setup)
+    True
+    """
+    app.add_config_value(
+        "llms_generate_txt",
+        default=True,
+        rebuild="",
+        types=frozenset({bool}),
+        description="Enable llms.txt generation.",
+    )
+    app.add_config_value(
+        "llms_generate_full",
+        default=True,
+        rebuild="",
+        types=frozenset({bool}),
+        description="Enable llms-full.txt generation.",
+    )
+    app.add_config_value(
+        "llms_generate_json",
+        default=True,
+        rebuild="",
+        types=frozenset({bool}),
+        description="Enable docs.json agent manifest generation.",
+    )
+    app.add_config_value(
+        "llms_generate_md_twins",
+        default=True,
+        rebuild="",
+        types=frozenset({bool}),
+        description="Enable per-page .md twin file generation.",
+    )
+    app.add_config_value(
+        "llms_txt_filename",
+        default="llms.txt",
+        rebuild="",
+        types=frozenset({str}),
+        description="Output filename for the llms.txt index.",
+    )
+    app.add_config_value(
+        "llms_full_filename",
+        default="llms-full.txt",
+        rebuild="",
+        types=frozenset({str}),
+        description="Output filename for the concatenated full-content file.",
+    )
+    app.add_config_value(
+        "llms_json_filename",
+        default="docs.json",
+        rebuild="",
+        types=frozenset({str}),
+        description="Output filename for the docs.json agent manifest.",
+    )
+    app.add_config_value(
+        "llms_excludes",
+        default=[],
+        rebuild="",
+        types=frozenset({list}),
+        description=(
+            "fnmatch patterns matched against each page's relative URL. "
+            "Matched pages are excluded from all LLM outputs."
+        ),
+    )
+    app.add_config_value(
+        "llms_description_length",
+        default=200,
+        rebuild="",
+        types=frozenset({int}),
+        description="Maximum character length for page descriptions.",
+    )
+
+    with contextlib.suppress(ExtensionError):
+        app.add_config_value(
+            "site_url",
+            default=None,
+            rebuild="",
+            types=frozenset({str, type(None)}),
+            description=(
+                "Site base URL — registered defensively; "
+                "sphinx-gp-sitemap usually registers this first."
+            ),
+        )
+
+    app.connect("build-finished", _write_llm_outputs)
+    app.connect("html-page-context", _inject_llms_context)
+
+    return {
+        "version": _EXTENSION_VERSION,
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }
+
+
+def _resolve_site_url(app: Sphinx) -> str | None:
+    """Resolve site URL from config, normalizing trailing slash."""
+    url: str | None = getattr(app.config, "site_url", None) or getattr(
+        app.config, "html_baseurl", None
+    )
+    if not url:
+        return None
+    return url if url.endswith("/") else url + "/"
+
+
+def _write_llm_outputs(app: Sphinx, exception: BaseException | None) -> None:
+    """Generate all enabled LLM output files at build-finished."""
+    if exception is not None:
+        return
+
+    if not hasattr(app.builder, "get_target_uri"):
+        return
+
+    site_url = _resolve_site_url(app)
+    if not site_url:
+        logger.info(
+            "sphinx-gp-llms: skipped — site_url and html_baseurl both unset",
+            type="llms",
+            subtype="configuration",
+        )
+        return
+
+    if app.config.llms_generate_txt:
+        from sphinx_gp_llms._llms_txt import write_llms_txt
+
+        write_llms_txt(app, site_url)
+
+    if app.config.llms_generate_full:
+        from sphinx_gp_llms._llms_full_txt import write_llms_full_txt
+
+        write_llms_full_txt(app, site_url)
+
+    if app.config.llms_generate_json:
+        from sphinx_gp_llms._docs_json import write_docs_json
+
+        write_docs_json(app, site_url)
+
+    if app.config.llms_generate_md_twins:
+        from sphinx_gp_llms._md_twins import write_md_twins
+
+        write_md_twins(app)
+
+
+def _inject_llms_context(
+    app: Sphinx,
+    pagename: str,
+    templatename: str,
+    context: dict[str, t.Any],
+    doctree: nodes.document | None,
+) -> None:
+    """Add LLM output link variables to the Jinja2 template context."""
+    del templatename, doctree
+
+    site_url = _resolve_site_url(app)
+    if not site_url:
+        return
+
+    if app.config.llms_generate_md_twins:
+        context["llms_md_url"] = pagename + ".md"
+    if app.config.llms_generate_txt:
+        context["llms_txt_url"] = app.config.llms_txt_filename
+    if app.config.llms_generate_full:
+        context["llms_full_url"] = app.config.llms_full_filename
+    if app.config.llms_generate_json:
+        context["llms_json_url"] = app.config.llms_json_filename

sphinx_gp_llms-0.0.1a24/src/sphinx_gp_llms/_description.py

@@ -0,0 +1,87 @@
+"""First-paragraph extraction from Sphinx doctrees.
+
+Provides a lightweight description extractor that walks a doctree and
+returns the text of the first body paragraph, suitable for use in
+``llms.txt`` link descriptions and ``docs.json`` page summaries.
+
+Examples
+--------
+>>> from sphinx_gp_llms._description import get_first_paragraph
+>>> callable(get_first_paragraph)
+True
+"""
+
+from __future__ import annotations
+
+import typing as t
+
+from docutils import nodes
+
+if t.TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+_SKIP_PARENTS = (
+    nodes.Admonition,
+    nodes.field_list,
+    nodes.sidebar,
+    nodes.topic,
+    nodes.comment,
+    nodes.footnote,
+)
+
+
+def _is_body_paragraph(node: nodes.paragraph) -> bool:
+    """Return True when *node* is a direct section-child paragraph."""
+    parent = node.parent
+    while parent is not None:
+        if isinstance(parent, _SKIP_PARENTS):
+            return False
+        if isinstance(parent, nodes.section):
+            return True
+        parent = parent.parent
+    return True
+
+
+def get_first_paragraph(
+    app: Sphinx,
+    docname: str,
+    max_length: int = 200,
+) -> str:
+    """Extract the first body paragraph from a page's doctree.
+
+    Parameters
+    ----------
+    app : Sphinx
+        Sphinx application instance.
+    docname : str
+        Document name (without extension).
+    max_length : int
+        Maximum characters to return.
+
+    Returns
+    -------
+    str
+        Flattened paragraph text, truncated with ``...`` when exceeding
+        *max_length*.
+
+    Examples
+    --------
+    >>> from sphinx_gp_llms._description import get_first_paragraph
+    >>> callable(get_first_paragraph)
+    True
+    """
+    doctree = app.env.get_doctree(docname)
+    title_text = ""
+    if docname in app.env.titles:
+        title_text = app.env.titles[docname].astext()
+
+    for node in doctree.findall(nodes.paragraph):
+        if not _is_body_paragraph(node):
+            continue
+        text = node.astext().replace("\n", " ").strip()
+        if not text or text == title_text:
+            continue
+        if len(text) > max_length:
+            return text[: max_length - 3] + "..."
+        return text
+    return ""

sphinx_gp_llms-0.0.1a24/src/sphinx_gp_llms/_docs_json.py

@@ -0,0 +1,188 @@
+"""Generate ``docs.json`` — an agent-oriented documentation manifest.
+
+Follows the agent-manifest convention established by Lakebed (Ping,
+``github.com/pingdotgg/span``).  The manifest provides structured
+metadata including ``agentEntrypoints``, a flat ``pages[]`` array with
+per-page ``markdownUrl`` and ``headings[]`` outlines.
+
+Examples
+--------
+>>> from sphinx_gp_llms._docs_json import write_docs_json
+>>> callable(write_docs_json)
+True
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import json
+import pathlib
+import typing as t
+
+from docutils import nodes
+from sphinx import addnodes
+from sphinx.util.logging import getLogger
+
+from sphinx_gp_llms._description import get_first_paragraph
+from sphinx_gp_llms._toctree import extract_toctree_sections
+
+if t.TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+logger = getLogger(__name__)
+
+
+class _Heading(t.TypedDict):
+    id: str
+    level: int
+    text: str
+
+
+class _Page(t.TypedDict):
+    title: str
+    description: str
+    section: str
+    url: str
+    markdownUrl: str
+    headings: list[_Heading]
+
+
+class _AgentEntrypoints(t.TypedDict):
+    manifest: str
+    llms: str
+    llmsFull: str
+
+
+class _DocsManifest(t.TypedDict):
+    name: str
+    url: str
+    description: str
+    sourceRepository: str
+    agentEntrypoints: _AgentEntrypoints
+    pages: list[_Page]
+
+
+def write_docs_json(app: Sphinx, site_url: str) -> None:
+    """Write ``docs.json`` to the build output directory.
+
+    Parameters
+    ----------
+    app : Sphinx
+        Sphinx application instance.
+    site_url : str
+        Normalized site base URL with trailing slash.
+
+    Examples
+    --------
+    >>> from sphinx_gp_llms._docs_json import write_docs_json
+    >>> callable(write_docs_json)
+    True
+    """
+    excludes: list[str] = list(app.config.llms_excludes)
+    sections = extract_toctree_sections(app)
+
+    section_map: dict[str, str] = {}
+    for section in sections:
+        caption = section.caption or "Documentation"
+        for docname in section.docnames:
+            section_map[docname] = caption
+
+    pages: list[_Page] = []
+    for docname in sorted(app.env.found_docs):
+        uri = app.builder.get_target_uri(docname)
+        if _is_excluded(uri, excludes):
+            continue
+
+        title_node = app.env.titles.get(docname)
+        if title_node is None:
+            continue
+        title = title_node.astext()
+        desc = get_first_paragraph(app, docname, app.config.llms_description_length)
+        headings = _extract_headings(app, docname)
+
+        pages.append(
+            _Page(
+                title=title,
+                description=desc,
+                section=section_map.get(docname, ""),
+                url="/" + uri,
+                markdownUrl="/" + docname + ".md",
+                headings=headings,
+            )
+        )
+
+    source_repo = _get_source_repository(app)
+    root_desc = get_first_paragraph(
+        app, app.config.root_doc, app.config.llms_description_length
+    )
+
+    manifest = _DocsManifest(
+        name=app.config.project,
+        url=site_url.rstrip("/"),
+        description=root_desc,
+        sourceRepository=source_repo,
+        agentEntrypoints=_AgentEntrypoints(
+            manifest="/" + app.config.llms_json_filename,
+            llms="/" + app.config.llms_txt_filename,
+            llmsFull="/" + app.config.llms_full_filename,
+        ),
+        pages=pages,
+    )
+
+    output = pathlib.Path(app.outdir) / app.config.llms_json_filename
+    output.write_text(
+        json.dumps(manifest, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
+    logger.info(
+        "sphinx-gp-llms: %s generated at %s",
+        app.config.llms_json_filename,
+        output,
+        type="llms",
+        subtype="information",
+    )
+
+
+def _extract_headings(app: Sphinx, docname: str) -> list[_Heading]:
+    """Extract heading id/level/text from the table-of-contents tree."""
+    toc = app.env.tocs.get(docname)
+    if toc is None:
+        return []
+    headings: list[_Heading] = []
+    _walk_toc(toc, level=1, headings=headings)
+    return headings
+
+
+def _walk_toc(
+    node: nodes.Node,
+    level: int,
+    headings: list[_Heading],
+) -> None:
+    """Recursively walk a toc bullet_list, collecting headings."""
+    if isinstance(node, nodes.bullet_list):
+        for item in node.children:
+            _walk_toc(item, level, headings)
+    elif isinstance(node, nodes.list_item):
+        for child in node.children:
+            if isinstance(child, addnodes.compact_paragraph):
+                for ref in child.findall(nodes.reference):
+                    anchor = ref.get("anchorname", "")
+                    text = ref.astext()
+                    heading_id = anchor.lstrip("#") if anchor else ""
+                    if text:
+                        headings.append(_Heading(id=heading_id, level=level, text=text))
+            elif isinstance(child, nodes.bullet_list):
+                _walk_toc(child, level + 1, headings)
+
+
+def _get_source_repository(app: Sphinx) -> str:
+    """Read source_repository from theme options."""
+    theme_opts = getattr(app.config, "html_theme_options", None)
+    if isinstance(theme_opts, dict):
+        return str(theme_opts.get("source_repository", ""))
+    return ""
+
+
+def _is_excluded(uri: str, patterns: list[str]) -> bool:
+    """Return True when *uri* matches any fnmatch pattern."""
+    return any(fnmatch.fnmatch(uri, p) for p in patterns)

sphinx_gp_llms-0.0.1a24/src/sphinx_gp_llms/_llms_full_txt.py

@@ -0,0 +1,84 @@
+"""Generate ``llms-full.txt`` — concatenated full-content Markdown.
+
+Community convention adopted by Anthropic, Cloudflare, Mintlify, and
+GitBook.  Each page's source content is included under a title header
+with a source URL reference, separated by ``---`` dividers.
+
+Examples
+--------
+>>> from sphinx_gp_llms._llms_full_txt import write_llms_full_txt
+>>> callable(write_llms_full_txt)
+True
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import pathlib
+import typing as t
+
+from sphinx.util.logging import getLogger
+
+if t.TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+logger = getLogger(__name__)
+
+
+def write_llms_full_txt(app: Sphinx, site_url: str) -> None:
+    """Write ``llms-full.txt`` to the build output directory.
+
+    Parameters
+    ----------
+    app : Sphinx
+        Sphinx application instance.
+    site_url : str
+        Normalized site base URL with trailing slash.
+
+    Examples
+    --------
+    >>> from sphinx_gp_llms._llms_full_txt import write_llms_full_txt
+    >>> callable(write_llms_full_txt)
+    True
+    """
+    excludes: list[str] = list(app.config.llms_excludes)
+    parts: list[str] = []
+
+    for docname in sorted(app.env.found_docs):
+        uri = app.builder.get_target_uri(docname)
+        if _is_excluded(uri, excludes):
+            continue
+
+        title_node = app.env.titles.get(docname)
+        title = title_node.astext() if title_node is not None else docname
+        url = site_url + uri
+        source_path = pathlib.Path(app.env.doc2path(docname))
+
+        parts.append(f"# {title}")
+        parts.append(f"Source: {url}")
+        parts.append("")
+
+        try:
+            content = source_path.read_text(encoding="utf-8")
+            parts.append(content.rstrip())
+        except (OSError, UnicodeDecodeError):
+            parts.append(f"(source not available for {docname})")
+
+        parts.append("")
+        parts.append("---")
+        parts.append("")
+
+    output = pathlib.Path(app.outdir) / app.config.llms_full_filename
+    output.write_text("\n".join(parts), encoding="utf-8")
+    logger.info(
+        "sphinx-gp-llms: %s generated at %s",
+        app.config.llms_full_filename,
+        output,
+        type="llms",
+        subtype="information",
+    )
+
+
+def _is_excluded(uri: str, patterns: list[str]) -> bool:
+    """Return True when *uri* matches any fnmatch pattern."""
+    return any(fnmatch.fnmatch(uri, p) for p in patterns)

sphinx_gp_llms-0.0.1a24/src/sphinx_gp_llms/_llms_txt.py

@@ -0,0 +1,96 @@
+"""Generate ``llms.txt`` — a structured Markdown index for LLM agents.
+
+Follows the specification at https://llmstxt.org/ (Jeremy Howard,
+Answer.AI, September 2024).  The file uses H1 for the project name,
+a blockquote summary, and H2 sections of bulleted ``[title](url)``
+links grouped by toctree caption.
+
+Examples
+--------
+>>> from sphinx_gp_llms._llms_txt import write_llms_txt
+>>> callable(write_llms_txt)
+True
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import pathlib
+import typing as t
+
+from sphinx.util.logging import getLogger
+
+from sphinx_gp_llms._description import get_first_paragraph
+from sphinx_gp_llms._toctree import extract_toctree_sections
+
+if t.TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+logger = getLogger(__name__)
+
+
+def write_llms_txt(app: Sphinx, site_url: str) -> None:
+    """Write ``llms.txt`` to the build output directory.
+
+    Parameters
+    ----------
+    app : Sphinx
+        Sphinx application instance.
+    site_url : str
+        Normalized site base URL with trailing slash.
+
+    Examples
+    --------
+    >>> from sphinx_gp_llms._llms_txt import write_llms_txt
+    >>> callable(write_llms_txt)
+    True
+    """
+    excludes: list[str] = list(app.config.llms_excludes)
+    sections = extract_toctree_sections(app)
+    lines: list[str] = []
+
+    lines.append(f"# {app.config.project}")
+    lines.append("")
+
+    max_len: int = app.config.llms_description_length
+    desc = get_first_paragraph(app, app.config.root_doc, max_len)
+    if desc:
+        lines.append(f"> {desc}")
+        lines.append("")
+
+    for section in sections:
+        section_name = section.caption or "Documentation"
+        lines.append(f"## {section_name}")
+        lines.append("")
+        for docname in section.docnames:
+            uri = app.builder.get_target_uri(docname)
+            if _is_excluded(uri, excludes):
+                continue
+            title_node = app.env.titles.get(docname)
+            if title_node is None:
+                continue
+            title = title_node.astext()
+            url = site_url + uri
+            page_desc = get_first_paragraph(
+                app, docname, app.config.llms_description_length
+            )
+            entry = f"- [{title}]({url})"
+            if page_desc:
+                entry += f": {page_desc}"
+            lines.append(entry)
+        lines.append("")
+
+    output = pathlib.Path(app.outdir) / app.config.llms_txt_filename
+    output.write_text("\n".join(lines), encoding="utf-8")
+    logger.info(
+        "sphinx-gp-llms: %s generated at %s",
+        app.config.llms_txt_filename,
+        output,
+        type="llms",
+        subtype="information",
+    )
+
+
+def _is_excluded(uri: str, patterns: list[str]) -> bool:
+    """Return True when *uri* matches any fnmatch pattern."""
+    return any(fnmatch.fnmatch(uri, p) for p in patterns)

sphinx_gp_llms-0.0.1a24/src/sphinx_gp_llms/_md_twins.py

@@ -0,0 +1,72 @@
+"""Generate per-page ``.md`` twin files alongside HTML output.
+
+Implements the per-page Markdown endpoint convention popularized by
+Mintlify, Cloudflare ("Markdown for Agents"), Stripe, and Vercel.
+Each HTML page at ``/path/page.html`` gets a Markdown sibling at
+``/path/page.md`` containing the original source content.
+
+Examples
+--------
+>>> from sphinx_gp_llms._md_twins import write_md_twins
+>>> callable(write_md_twins)
+True
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import pathlib
+import shutil
+import typing as t
+
+from sphinx.util.logging import getLogger
+
+if t.TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+logger = getLogger(__name__)
+
+
+def write_md_twins(app: Sphinx) -> None:
+    """Copy source files as ``.md`` siblings in the build output directory.
+
+    Parameters
+    ----------
+    app : Sphinx
+        Sphinx application instance.
+
+    Examples
+    --------
+    >>> from sphinx_gp_llms._md_twins import write_md_twins
+    >>> callable(write_md_twins)
+    True
+    """
+    excludes: list[str] = list(app.config.llms_excludes)
+    outdir = pathlib.Path(app.outdir)
+    count = 0
+
+    for docname in sorted(app.env.found_docs):
+        uri = app.builder.get_target_uri(docname)
+        if _is_excluded(uri, excludes):
+            continue
+
+        source_path = pathlib.Path(app.env.doc2path(docname))
+        if not source_path.exists():
+            continue
+
+        target = outdir / (docname + ".md")
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source_path, target)
+        count += 1
+
+    logger.info(
+        "sphinx-gp-llms: %d .md twin files written",
+        count,
+        type="llms",
+        subtype="information",
+    )
+
+
+def _is_excluded(uri: str, patterns: list[str]) -> bool:
+    """Return True when *uri* matches any fnmatch pattern."""
+    return any(fnmatch.fnmatch(uri, p) for p in patterns)

sphinx_gp_llms-0.0.1a24/src/sphinx_gp_llms/_toctree.py

@@ -0,0 +1,79 @@
+"""Toctree section extraction for llms.txt grouping.
+
+Walks the root document's doctree to find ``toctree`` directives and
+their ``:caption:`` options, producing a flat list of sections suitable
+for the H2-delimited structure of ``llms.txt``.
+
+Examples
+--------
+>>> from sphinx_gp_llms._toctree import ToctreeSection
+>>> s = ToctreeSection(caption="Guide", docnames=["quickstart"])
+>>> s.caption
+'Guide'
+"""
+
+from __future__ import annotations
+
+import typing as t
+
+from sphinx import addnodes
+
+if t.TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+
+class ToctreeSection(t.NamedTuple):
+    """One section of pages grouped by toctree caption.
+
+    Examples
+    --------
+    >>> ToctreeSection(caption="API", docnames=["api/index"])
+    ToctreeSection(caption='API', docnames=['api/index'])
+    """
+
+    caption: str | None
+    docnames: list[str]
+
+
+def extract_toctree_sections(app: Sphinx) -> list[ToctreeSection]:
+    """Walk the root document's toctree nodes and group pages by caption.
+
+    Parameters
+    ----------
+    app : Sphinx
+        Sphinx application instance (must have a built environment).
+
+    Returns
+    -------
+    list[ToctreeSection]
+        Sections in document order.  Pages not referenced by any
+        toctree in the root document get a ``caption=None`` fallback
+        section at the end.
+
+    Examples
+    --------
+    >>> from sphinx_gp_llms._toctree import extract_toctree_sections
+    >>> callable(extract_toctree_sections)
+    True
+    """
+    root_doc = app.config.root_doc
+    doctree = app.env.get_doctree(root_doc)
+
+    sections: list[ToctreeSection] = []
+    assigned: set[str] = set()
+
+    for toctree_node in doctree.findall(addnodes.toctree):
+        caption = toctree_node.get("caption")
+        docnames: list[str] = []
+        for _title, docname in toctree_node["entries"]:
+            if docname and docname in app.env.found_docs and docname not in assigned:
+                docnames.append(docname)
+                assigned.add(docname)
+        if docnames:
+            sections.append(ToctreeSection(caption=caption, docnames=docnames))
+
+    remaining = sorted(app.env.found_docs - assigned - {root_doc})
+    if remaining:
+        sections.append(ToctreeSection(caption=None, docnames=remaining))
+
+    return sections