llms-ink 0.1.0__tar.gz

llms_ink-0.1.0/.gitignore

@@ -0,0 +1,5 @@
+.venv/
+dist/
+__pycache__/
+*.egg-info/
+.pytest_cache/

llms_ink-0.1.0/CHANGELOG.md

@@ -0,0 +1,9 @@
+# Changelog
+
+## 0.1.0 — 2026-11-14
+
+- `Client.llms(url, mode, fresh)` and `Client.check(url)` for the llms.ink JSON API, standard library only.
+- Optional API key from `key=` or `LLMS_INK_KEY`, sent as `Authorization: Bearer`.
+- Typed errors from problem details: `InvalidKey`, `RateLimited`, `QuotaExceeded` (with `retry_after`), `LlmsInkError`.
+- `llms_ink.langchain.LlmsInkLoader` (`[langchain]`) and `llms_ink.llama_index.LlmsInkReader` (`[llama-index]`).
+- `python -m llms_ink <url> [--full] [--fresh] [--json]`.

llms_ink-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Lucas Espindola
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

llms_ink-0.1.0/PKG-INFO

@@ -0,0 +1,134 @@
+Metadata-Version: 2.5
+Name: llms-ink
+Version: 0.1.0
+Summary: Turn any link — site, docs section, page, GitHub repo, PDF or video — into llms.txt and llms-full.txt. Python client for llms.ink, with LangChain and LlamaIndex loaders.
+Project-URL: Homepage, https://llms.ink
+Project-URL: Documentation, https://llms.ink/docs/
+Author: Lucas Espindola
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,context,docs,langchain,llama-index,llm,llms-full.txt,llms.txt,markdown,rag
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.2; extra == 'langchain'
+Provides-Extra: llama-index
+Requires-Dist: llama-index-core>=0.10; extra == 'llama-index'
+Description-Content-Type: text/markdown
+
+# llms-ink
+
+Turn any public link — a site, docs section, page, GitHub repo, PDF or video — into
+[llms.txt](https://llmstxt.org) or llms-full.txt, from Python. A client for [llms.ink](https://llms.ink)
+with LangChain and LlamaIndex loaders. No required dependencies.
+
+```sh
+pip install llms-ink
+```
+
+## Use
+
+```python
+from llms_ink import Client
+
+ink = Client()
+index = ink.llms("llmstxt.org")                         # llms.txt: an index of links
+docs = ink.llms("docs.astro.build/en/guides/", mode="full")  # llms-full.txt: the content
+
+print(docs.title, docs.kind, docs.tokens)
+print(docs.text)
+```
+
+`llms()` returns a `Result`: `url`, `kind` (`site`, `section`, `page`, `repo`, `document`, `video`),
+`source` (`native` = published by the site, `generated` = made by llms.ink), `mode`, `title`, `text`,
+`tokens`, `pages`, `model`, `cached`, `generated_at`, and `media` for videos (duration, downloads,
+transcript segments). `fresh=True` skips the cache and rebuilds.
+
+Check a site's published llms.txt:
+
+```python
+report = ink.check("llmstxt.org")
+print(report.found, report.summary.errors, [i.message for i in report.issues])
+```
+
+## API key (optional)
+
+Without a key you share the free anonymous limits (10 requests/minute, 60 new links a day per IP;
+cached links are free). A key gets its own daily quota:
+
+```sh
+export LLMS_INK_KEY=lk_...
+```
+
+or `Client(key="lk_...")`. It is sent as `Authorization: Bearer`. A bad key raises `InvalidKey`; it never
+falls back to anonymous. [Ask for a key](https://lucasespindola.com/message?text=I%27d+like+an+llms.ink+API+key).
+
+## Errors
+
+```python
+from llms_ink import Client, LlmsInkError, QuotaExceeded, RateLimited
+
+try:
+    Client().llms("example.com")
+except (RateLimited, QuotaExceeded) as err:
+    print(f"wait {err.retry_after}s")
+except LlmsInkError as err:
+    print(err.status, err.code, err.detail)
+```
+
+`LlmsInkError` carries the API's `status`, `code` (see the [error table](https://llms.ink/docs/#errors)),
+`detail` and `errors`. `InvalidKey`, `RateLimited` and `QuotaExceeded` are subclasses.
+
+## LangChain
+
+```sh
+pip install 'llms-ink[langchain]'
+```
+
+```python
+from llms_ink.langchain import LlmsInkLoader
+
+docs = LlmsInkLoader(["llmstxt.org", "github.com/AnswerDotAI/llms-txt"], mode="full").load()
+docs[0].metadata  # {'url', 'title', 'kind', 'source', 'mode', 'tokens', 'generated_at'}
+```
+
+## LlamaIndex
+
+```sh
+pip install 'llms-ink[llama-index]'
+```
+
+```python
+from llama_index.core import VectorStoreIndex
+from llms_ink.llama_index import LlmsInkReader
+
+docs = LlmsInkReader(mode="full").load_data(["docs.astro.build/en/guides/"])
+index = VectorStoreIndex.from_documents(docs)
+```
+
+## Command line
+
+```sh
+python -m llms_ink llmstxt.org            # llms.txt
+python -m llms_ink llmstxt.org --full     # llms-full.txt
+python -m llms_ink llmstxt.org --json     # the whole result
+```
+
+## Development
+
+```sh
+python3 -m venv .venv && .venv/bin/pip install -e '.[langchain,llama-index]' pytest
+.venv/bin/pytest                # against a local stub server
+.venv/bin/pytest -m live        # one real call to https://llms.ink
+```
+
+`LLMS_INK_API` points the client at another API base URL.
+
+MIT licensed.

llms_ink-0.1.0/README.md

@@ -0,0 +1,109 @@
+# llms-ink
+
+Turn any public link — a site, docs section, page, GitHub repo, PDF or video — into
+[llms.txt](https://llmstxt.org) or llms-full.txt, from Python. A client for [llms.ink](https://llms.ink)
+with LangChain and LlamaIndex loaders. No required dependencies.
+
+```sh
+pip install llms-ink
+```
+
+## Use
+
+```python
+from llms_ink import Client
+
+ink = Client()
+index = ink.llms("llmstxt.org")                         # llms.txt: an index of links
+docs = ink.llms("docs.astro.build/en/guides/", mode="full")  # llms-full.txt: the content
+
+print(docs.title, docs.kind, docs.tokens)
+print(docs.text)
+```
+
+`llms()` returns a `Result`: `url`, `kind` (`site`, `section`, `page`, `repo`, `document`, `video`),
+`source` (`native` = published by the site, `generated` = made by llms.ink), `mode`, `title`, `text`,
+`tokens`, `pages`, `model`, `cached`, `generated_at`, and `media` for videos (duration, downloads,
+transcript segments). `fresh=True` skips the cache and rebuilds.
+
+Check a site's published llms.txt:
+
+```python
+report = ink.check("llmstxt.org")
+print(report.found, report.summary.errors, [i.message for i in report.issues])
+```
+
+## API key (optional)
+
+Without a key you share the free anonymous limits (10 requests/minute, 60 new links a day per IP;
+cached links are free). A key gets its own daily quota:
+
+```sh
+export LLMS_INK_KEY=lk_...
+```
+
+or `Client(key="lk_...")`. It is sent as `Authorization: Bearer`. A bad key raises `InvalidKey`; it never
+falls back to anonymous. [Ask for a key](https://lucasespindola.com/message?text=I%27d+like+an+llms.ink+API+key).
+
+## Errors
+
+```python
+from llms_ink import Client, LlmsInkError, QuotaExceeded, RateLimited
+
+try:
+    Client().llms("example.com")
+except (RateLimited, QuotaExceeded) as err:
+    print(f"wait {err.retry_after}s")
+except LlmsInkError as err:
+    print(err.status, err.code, err.detail)
+```
+
+`LlmsInkError` carries the API's `status`, `code` (see the [error table](https://llms.ink/docs/#errors)),
+`detail` and `errors`. `InvalidKey`, `RateLimited` and `QuotaExceeded` are subclasses.
+
+## LangChain
+
+```sh
+pip install 'llms-ink[langchain]'
+```
+
+```python
+from llms_ink.langchain import LlmsInkLoader
+
+docs = LlmsInkLoader(["llmstxt.org", "github.com/AnswerDotAI/llms-txt"], mode="full").load()
+docs[0].metadata  # {'url', 'title', 'kind', 'source', 'mode', 'tokens', 'generated_at'}
+```
+
+## LlamaIndex
+
+```sh
+pip install 'llms-ink[llama-index]'
+```
+
+```python
+from llama_index.core import VectorStoreIndex
+from llms_ink.llama_index import LlmsInkReader
+
+docs = LlmsInkReader(mode="full").load_data(["docs.astro.build/en/guides/"])
+index = VectorStoreIndex.from_documents(docs)
+```
+
+## Command line
+
+```sh
+python -m llms_ink llmstxt.org            # llms.txt
+python -m llms_ink llmstxt.org --full     # llms-full.txt
+python -m llms_ink llmstxt.org --json     # the whole result
+```
+
+## Development
+
+```sh
+python3 -m venv .venv && .venv/bin/pip install -e '.[langchain,llama-index]' pytest
+.venv/bin/pytest                # against a local stub server
+.venv/bin/pytest -m live        # one real call to https://llms.ink
+```
+
+`LLMS_INK_API` points the client at another API base URL.
+
+MIT licensed.

llms_ink-0.1.0/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "llms-ink"
+dynamic = ["version"]
+description = "Turn any link — site, docs section, page, GitHub repo, PDF or video — into llms.txt and llms-full.txt. Python client for llms.ink, with LangChain and LlamaIndex loaders."
+readme = "README.md"
+requires-python = ">=3.9"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Lucas Espindola" }]
+keywords = ["llms.txt", "llms-full.txt", "llm", "ai", "markdown", "docs", "context", "rag", "langchain", "llama-index"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Topic :: Text Processing :: Markup :: Markdown",
+  "Typing :: Typed",
+]
+dependencies = []
+
+[project.optional-dependencies]
+langchain = ["langchain-core>=0.2"]
+llama-index = ["llama-index-core>=0.10"]
+
+[project.urls]
+Homepage = "https://llms.ink"
+Documentation = "https://llms.ink/docs/"
+
+[tool.hatch.version]
+path = "src/llms_ink/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/llms_ink"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src", "tests", "README.md", "LICENSE", "CHANGELOG.md"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-m 'not live'"
+markers = ["live: calls the real https://llms.ink (run with: pytest -m live)"]

llms_ink-0.1.0/src/llms_ink/__init__.py

@@ -0,0 +1,41 @@
+"""Turn any link — site, docs section, page, GitHub repo, PDF or video — into llms.txt and llms-full.txt.
+
+    >>> from llms_ink import Client
+    >>> Client().llms("llmstxt.org").text
+"""
+
+__version__ = "0.1.0"
+
+from .client import (  # noqa: E402
+    DEFAULT_BASE_URL,
+    BrokenLink,
+    CheckIssue,
+    CheckLinks,
+    CheckResult,
+    CheckSummary,
+    Client,
+    Media,
+    Result,
+    Segment,
+    Transcript,
+)
+from .errors import InvalidKey, LlmsInkError, QuotaExceeded, RateLimited  # noqa: E402
+
+__all__ = [
+    "DEFAULT_BASE_URL",
+    "BrokenLink",
+    "CheckIssue",
+    "CheckLinks",
+    "CheckResult",
+    "CheckSummary",
+    "Client",
+    "InvalidKey",
+    "LlmsInkError",
+    "Media",
+    "QuotaExceeded",
+    "RateLimited",
+    "Result",
+    "Segment",
+    "Transcript",
+    "__version__",
+]

llms_ink-0.1.0/src/llms_ink/__main__.py

@@ -0,0 +1,45 @@
+"""python -m llms_ink <url> [--full] [--fresh] [--json]: prints llms.txt (or llms-full.txt) for a link."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import List, Optional
+
+from . import __version__
+from .client import Client
+from .errors import LlmsInkError, QuotaExceeded, RateLimited
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m llms_ink",
+        description="Print llms.txt for any link: a site, docs section, page, GitHub repo, PDF or video.",
+        epilog="Uses LLMS_INK_KEY as the API key when set (optional).",
+    )
+    parser.add_argument("url", help="any public link, e.g. llmstxt.org")
+    parser.add_argument("--full", action="store_true", help="llms-full.txt (the content) instead of the index")
+    parser.add_argument("--fresh", action="store_true", help="skip the cache and rebuild")
+    parser.add_argument("--json", action="store_true", help="the whole API result as JSON")
+    parser.add_argument("--version", action="version", version=f"llms-ink {__version__}")
+    args = parser.parse_args(argv)
+
+    try:
+        result = Client().llms(args.url, mode="full" if args.full else "llms", fresh=args.fresh)
+    except LlmsInkError as err:
+        message = err.detail
+        if isinstance(err, (RateLimited, QuotaExceeded)) and err.retry_after is not None:
+            message += f" Try again in {err.retry_after}s."
+        print(f"llms-ink: {message}", file=sys.stderr)
+        return 1
+
+    if args.json:
+        print(json.dumps(result.raw, indent=2, ensure_ascii=False))
+    else:
+        sys.stdout.write(result.text if result.text.endswith("\n") else result.text + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

llms_ink-0.1.0/src/llms_ink/client.py

@@ -0,0 +1,289 @@
+"""The llms.ink JSON API (https://llms.ink/docs/) over the standard library's urllib."""
+
+from __future__ import annotations
+
+import json
+import os
+import socket
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from typing import Any, Dict, List, Optional
+
+from . import __version__
+from .errors import LlmsInkError, error_for
+
+DEFAULT_BASE_URL = "https://llms.ink"
+MODES = ("llms", "full")
+
+
+@dataclass
+class Segment:
+    """A transcript line; ``start`` and ``end`` are seconds into the source video."""
+
+    start: float
+    end: float
+    text: str
+
+
+@dataclass
+class Transcript:
+    source: str
+    segments: List[Segment]
+
+
+@dataclass
+class Media:
+    """Video results only: what ``/api/v0/frame`` and ``/api/v0/media`` can serve."""
+
+    duration: Optional[float]
+    has_video: bool
+    has_audio: bool
+    thumbnail: Optional[str]
+    downloads: List[str]
+    is_live: Optional[bool] = None
+    transcript: Optional[Transcript] = None
+
+    @classmethod
+    def from_json(cls, d: Dict[str, Any]) -> "Media":
+        t = d.get("transcript")
+        return cls(
+            duration=d.get("duration"),
+            has_video=bool(d.get("hasVideo")),
+            has_audio=bool(d.get("hasAudio")),
+            thumbnail=d.get("thumbnail"),
+            downloads=list(d.get("downloads") or []),
+            is_live=d.get("isLive"),
+            transcript=Transcript(
+                source=t.get("source", ""),
+                segments=[Segment(s["start"], s["end"], s["text"]) for s in t.get("segments") or []],
+            )
+            if t
+            else None,
+        )
+
+
+@dataclass
+class Result:
+    """``GET /api/v0/llms``: the llms.txt or llms-full.txt of a link and what it was built from.
+
+    ``kind`` is ``site``, ``section``, ``page``, ``repo``, ``document`` or ``video``; ``source`` is ``native``
+    (published by the site) or ``generated`` (made by llms.ink). ``text`` is the Markdown file.
+    """
+
+    url: str
+    kind: str
+    source: str
+    mode: str
+    title: str
+    text: str
+    tokens: int
+    pages: int
+    model: Optional[str]
+    cached: bool
+    generated_at: str
+    media: Optional[Media] = None
+    raw: Dict[str, Any] = field(default_factory=dict, repr=False, compare=False)
+
+    @classmethod
+    def from_json(cls, d: Dict[str, Any]) -> "Result":
+        return cls(
+            url=d.get("url", ""),
+            kind=d.get("kind", ""),
+            source=d.get("source", ""),
+            mode=d.get("mode", ""),
+            title=d.get("title", ""),
+            text=d.get("text", ""),
+            tokens=d.get("tokens", 0),
+            pages=d.get("pages", 0),
+            model=d.get("model"),
+            cached=bool(d.get("cached")),
+            generated_at=d.get("generatedAt", ""),
+            media=Media.from_json(d["media"]) if d.get("media") else None,
+            raw=d,
+        )
+
+
+@dataclass
+class CheckIssue:
+    """``level`` is ``error``, ``warning`` or ``info``; ``code`` names the check (``missing_title``, ``broken_link``…)."""
+
+    level: str
+    code: str
+    line: Optional[int]
+    message: str
+
+
+@dataclass
+class BrokenLink:
+    url: str
+    status: int  # 0 when it didn't answer in time
+
+
+@dataclass
+class CheckLinks:
+    total: int
+    checked: int
+    broken: List[BrokenLink]
+
+
+@dataclass
+class CheckSummary:
+    errors: int
+    warnings: int
+
+
+@dataclass
+class CheckResult:
+    """``GET /api/v0/check``: a published llms.txt checked against the format and its links."""
+
+    site: str
+    url: str
+    found: bool
+    status: int
+    title: Optional[str]
+    tokens: int
+    sections: int
+    links: CheckLinks
+    llms_full: bool
+    issues: List[CheckIssue]
+    summary: CheckSummary
+    checked_at: str
+    raw: Dict[str, Any] = field(default_factory=dict, repr=False, compare=False)
+
+    @classmethod
+    def from_json(cls, d: Dict[str, Any]) -> "CheckResult":
+        links = d.get("links") or {}
+        summary = d.get("summary") or {}
+        return cls(
+            site=d.get("site", ""),
+            url=d.get("url", ""),
+            found=bool(d.get("found")),
+            status=d.get("status", 0),
+            title=d.get("title"),
+            tokens=d.get("tokens", 0),
+            sections=d.get("sections", 0),
+            links=CheckLinks(
+                total=links.get("total", 0),
+                checked=links.get("checked", 0),
+                broken=[BrokenLink(b["url"], b["status"]) for b in links.get("broken") or []],
+            ),
+            llms_full=bool(d.get("llmsFull")),
+            issues=[CheckIssue(i["level"], i["code"], i.get("line"), i["message"]) for i in d.get("issues") or []],
+            summary=CheckSummary(errors=summary.get("errors", 0), warnings=summary.get("warnings", 0)),
+            checked_at=d.get("checkedAt", ""),
+            raw=d,
+        )
+
+
+class Client:
+    """A client for https://llms.ink.
+
+    ``key`` is an optional API key (``lk_…``), sent as ``Authorization: Bearer``; it defaults to the
+    ``LLMS_INK_KEY`` environment variable. Without one, requests are anonymous and share the free limits.
+    ``base_url`` defaults to ``LLMS_INK_API`` or https://llms.ink. Building a big site or a video can take
+    minutes, hence the long default ``timeout`` (seconds).
+    """
+
+    def __init__(
+        self,
+        key: Optional[str] = None,
+        *,
+        base_url: Optional[str] = None,
+        timeout: float = 300.0,
+    ) -> None:
+        key = key if key is not None else os.environ.get("LLMS_INK_KEY")
+        self.key = key.strip() if key and key.strip() else None
+        self.base_url = (base_url or os.environ.get("LLMS_INK_API") or DEFAULT_BASE_URL).rstrip("/")
+        self.timeout = timeout
+
+    def __repr__(self) -> str:
+        return f"Client(base_url={self.base_url!r}, key={'set' if self.key else None!r})"
+
+    def llms(self, url: str, mode: str = "llms", fresh: bool = False) -> Result:
+        """llms.txt (``mode='llms'``, an index of links) or llms-full.txt (``mode='full'``, the content inlined)
+        for any public link. ``fresh=True`` skips the cache and rebuilds (counts against the daily quota)."""
+        if mode not in MODES:
+            raise ValueError(f"mode must be 'llms' (the index) or 'full' (llms-full.txt), not {mode!r}")
+        params = {"url": url, "mode": mode}
+        if fresh:
+            params["fresh"] = "1"
+        return Result.from_json(self._get("/api/v0/llms", params))
+
+    def check(self, url: str) -> CheckResult:
+        """Checks a site's published llms.txt against the format and that its links answer."""
+        return CheckResult.from_json(self._get("/api/v0/check", {"url": url}))
+
+    def _get(self, path: str, params: Dict[str, str]) -> Dict[str, Any]:
+        headers = {"Accept": "application/json", "User-Agent": f"llms-ink-python/{__version__}"}
+        if self.key:
+            headers["Authorization"] = f"Bearer {self.key}"
+        request = urllib.request.Request(f"{self.base_url}{path}?{urllib.parse.urlencode(params)}", headers=headers)
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout) as res:
+                body = res.read()
+        except urllib.error.HTTPError as err:
+            raise _problem(err) from None
+        except (urllib.error.URLError, socket.timeout, TimeoutError, ConnectionError) as err:
+            reason = getattr(err, "reason", err)
+            raise LlmsInkError(f"Couldn't reach {self.base_url}: {reason}") from err
+        try:
+            return json.loads(body)
+        except ValueError as err:
+            raise LlmsInkError(f"{self.base_url}{path} answered with something other than JSON.") from err
+
+
+def _problem(err: urllib.error.HTTPError) -> LlmsInkError:
+    """An HTTP error response → the typed error for its problem ``code``."""
+    raw = err.read()
+    try:
+        data = json.loads(raw)
+    except ValueError:
+        data = None
+    if not isinstance(data, dict):
+        data = {}
+    status = data.get("status") if isinstance(data.get("status"), int) else err.code
+    detail = data.get("detail") or data.get("title") or f"HTTP {err.code} {err.reason or 'error'}"
+    code = data.get("code")
+    return error_for(code)(
+        detail,
+        status=status,
+        code=code,
+        title=data.get("title"),
+        errors=data.get("errors"),
+        retry_after=_retry_after(err.headers.get("Retry-After") if err.headers else None),
+    )
+
+
+def _retry_after(value: Optional[str]) -> Optional[int]:
+    """``Retry-After`` as seconds: delta-seconds, or an HTTP date turned into seconds from now."""
+    if not value:
+        return None
+    value = value.strip()
+    if value.isdigit():
+        return int(value)
+    try:
+        when = parsedate_to_datetime(value)
+    except (TypeError, ValueError):
+        return None
+    if when is None:
+        return None
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=timezone.utc)
+    return max(0, int((when - datetime.now(timezone.utc)).total_seconds()))
+
+
+def _metadata(result: Result) -> Dict[str, Any]:
+    """Document metadata for the LangChain and LlamaIndex loaders."""
+    return {
+        "url": result.url,
+        "title": result.title,
+        "kind": result.kind,
+        "source": result.source,
+        "mode": result.mode,
+        "tokens": result.tokens,
+        "generated_at": result.generated_at,
+    }

llms_ink-0.1.0/src/llms_ink/errors.py

@@ -0,0 +1,56 @@
+"""Typed errors from the API's RFC 9457 problem details (codes at https://llms.ink/docs/#errors)."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+
+class LlmsInkError(Exception):
+    """An API error, or llms.ink couldn't be reached (then ``status`` and ``code`` are ``None``).
+
+    ``code`` is the problem's machine-readable code (``invalid_url``, ``unprocessable``, ``upstream_error``…),
+    ``detail`` the human-readable message, ``errors`` the per-parameter issues of ``invalid_request``.
+    ``retry_after`` is the ``Retry-After`` header in seconds, when the response had one.
+    """
+
+    def __init__(
+        self,
+        detail: str,
+        *,
+        status: Optional[int] = None,
+        code: Optional[str] = None,
+        title: Optional[str] = None,
+        errors: Optional[List[Dict[str, Any]]] = None,
+        retry_after: Optional[int] = None,
+    ) -> None:
+        super().__init__(detail)
+        self.detail = detail
+        self.status = status
+        self.code = code
+        self.title = title
+        self.errors = errors or []
+        self.retry_after = retry_after
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(status={self.status!r}, code={self.code!r}, detail={self.detail!r})"
+
+
+class InvalidKey(LlmsInkError):
+    """401 ``invalid_key``: the API key is unknown or revoked. Requests never fall back to anonymous."""
+
+
+class RateLimited(LlmsInkError):
+    """429 ``rate_limited``: too many requests this minute; wait ``retry_after`` seconds."""
+
+
+class QuotaExceeded(LlmsInkError):
+    """429 ``quota_exceeded``: the daily quota of new links is used up; ``retry_after`` is the seconds until it
+    resets (midnight UTC). Cached links still work."""
+
+
+_BY_CODE = {"invalid_key": InvalidKey, "rate_limited": RateLimited, "quota_exceeded": QuotaExceeded}
+
+
+def error_for(code: Optional[str]) -> type:
+    """The exception class for a problem ``code``."""
+    return _BY_CODE.get(code or "", LlmsInkError)

llms_ink-0.1.0/src/llms_ink/langchain.py

@@ -0,0 +1,44 @@
+"""LangChain document loader: ``pip install 'llms-ink[langchain]'``.
+
+    >>> from llms_ink.langchain import LlmsInkLoader
+    >>> docs = LlmsInkLoader(["docs.astro.build/en/guides/"], mode="full").load()
+"""
+
+from __future__ import annotations
+
+from typing import Iterator, Optional, Sequence, Union
+
+try:
+    from langchain_core.document_loaders import BaseLoader
+    from langchain_core.documents import Document
+except ImportError as err:  # pragma: no cover
+    raise ImportError("LlmsInkLoader needs langchain-core: pip install 'llms-ink[langchain]'") from err
+
+from .client import MODES, Client, _metadata
+
+
+class LlmsInkLoader(BaseLoader):
+    """One Document per link: llms.txt (``mode='llms'``) or llms-full.txt (``mode='full'``) as
+    ``page_content``, with metadata ``url``, ``title``, ``kind``, ``source``, ``mode``, ``tokens`` and
+    ``generated_at``. ``key`` defaults to ``LLMS_INK_KEY``; pass ``client`` to reuse one."""
+
+    def __init__(
+        self,
+        urls: Union[str, Sequence[str]],
+        mode: str = "llms",
+        *,
+        fresh: bool = False,
+        key: Optional[str] = None,
+        client: Optional[Client] = None,
+    ) -> None:
+        if mode not in MODES:
+            raise ValueError(f"mode must be 'llms' or 'full', not {mode!r}")
+        self.urls = [urls] if isinstance(urls, str) else list(urls)
+        self.mode = mode
+        self.fresh = fresh
+        self.client = client or Client(key)
+
+    def lazy_load(self) -> Iterator[Document]:
+        for url in self.urls:
+            result = self.client.llms(url, mode=self.mode, fresh=self.fresh)
+            yield Document(page_content=result.text, metadata=_metadata(result))

llms_ink-0.1.0/src/llms_ink/llama_index.py

@@ -0,0 +1,46 @@
+"""LlamaIndex reader: ``pip install 'llms-ink[llama-index]'``.
+
+    >>> from llms_ink.llama_index import LlmsInkReader
+    >>> docs = LlmsInkReader(mode="full").load_data(["docs.astro.build/en/guides/"])
+"""
+
+from __future__ import annotations
+
+from typing import Iterator, List, Optional, Union
+
+try:
+    from llama_index.core import Document
+    from llama_index.core.readers.base import BaseReader
+except ImportError as err:  # pragma: no cover
+    raise ImportError("LlmsInkReader needs llama-index-core: pip install 'llms-ink[llama-index]'") from err
+
+from .client import MODES, Client, _metadata
+
+
+class LlmsInkReader(BaseReader):
+    """One Document per link: llms.txt (``mode='llms'``) or llms-full.txt (``mode='full'``) as text, with
+    metadata ``url``, ``title``, ``kind``, ``source``, ``mode``, ``tokens`` and ``generated_at``.
+    ``key`` defaults to ``LLMS_INK_KEY``; pass ``client`` to reuse one."""
+
+    def __init__(
+        self,
+        mode: str = "llms",
+        *,
+        fresh: bool = False,
+        key: Optional[str] = None,
+        client: Optional[Client] = None,
+    ) -> None:
+        if mode not in MODES:
+            raise ValueError(f"mode must be 'llms' or 'full', not {mode!r}")
+        self.mode = mode
+        self.fresh = fresh
+        self.client = client or Client(key)
+
+    def lazy_load_data(self, urls: Union[str, List[str]], mode: Optional[str] = None) -> Iterator[Document]:
+        """Documents for ``urls``; ``mode`` overrides the reader's for this call."""
+        for url in [urls] if isinstance(urls, str) else urls:
+            result = self.client.llms(url, mode=mode or self.mode, fresh=self.fresh)
+            yield Document(text=result.text, metadata=_metadata(result))
+
+    def load_data(self, urls: Union[str, List[str]], mode: Optional[str] = None) -> List[Document]:
+        return list(self.lazy_load_data(urls, mode))

llms_ink-0.1.0/tests/conftest.py

@@ -0,0 +1,134 @@
+"""A local stub of the llms.ink API, answering like the real one for a few known links."""
+
+from __future__ import annotations
+
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any, Dict, List
+from urllib.parse import parse_qs, urlsplit
+
+import pytest
+
+LLMS = "# llms.txt\n\n> A proposal to standardise on using an /llms.txt file.\n\n## Docs\n\n- [Intro](https://llmstxt.org/intro.html)\n"
+FULL = "# llms.txt\n\n> A proposal.\n\n## Intro\n\nThe whole page.\n"
+GOOD_KEY = "lk_" + "a" * 32
+
+
+def result(url: str, mode: str) -> Dict[str, Any]:
+    body: Dict[str, Any] = {
+        "url": f"https://{url}/",
+        "kind": "site",
+        "source": "native",
+        "mode": mode,
+        "title": "llms.txt",
+        "text": LLMS if mode == "llms" else FULL,
+        "tokens": 42,
+        "pages": 1,
+        "model": None,
+        "cached": True,
+        "generatedAt": "2026-11-14T00:00:00.000Z",
+    }
+    if url.startswith("youtu.be"):
+        body.update(
+            kind="video",
+            source="generated",
+            model="gpt",
+            media={
+                "duration": 61.5,
+                "hasVideo": True,
+                "hasAudio": True,
+                "thumbnail": "https://i.ytimg.com/vi/x/hq.jpg",
+                "downloads": ["mp4", "m4a"],
+                "transcript": {"source": "captions", "segments": [{"start": 0, "end": 2.5, "text": "Hello"}]},
+            },
+        )
+    return body
+
+
+CHECK = {
+    "site": "https://llmstxt.org",
+    "url": "https://llmstxt.org/llms.txt",
+    "found": True,
+    "status": 200,
+    "title": "llms.txt",
+    "tokens": 300,
+    "sections": 2,
+    "links": {"total": 3, "checked": 3, "broken": [{"url": "https://llmstxt.org/gone", "status": 404}]},
+    "llmsFull": False,
+    "issues": [{"level": "error", "code": "broken_link", "line": 9, "message": "https://llmstxt.org/gone answers HTTP 404."}],
+    "summary": {"errors": 1, "warnings": 0},
+    "checkedAt": "2026-11-14T00:00:00.000Z",
+}
+
+PROBLEMS = {
+    "quota.test": (429, "quota_exceeded", "Daily quota of new links used up.", {"Retry-After": "3600"}),
+    "busy.test": (429, "rate_limited", "Too many requests this minute.", {"Retry-After": "12"}),
+    "login.test": (422, "unprocessable", "That page needs a login.", {}),
+}
+
+
+class Stub:
+    def __init__(self) -> None:
+        self.requests: List[Dict[str, Any]] = []
+        self.base_url = ""
+
+    @property
+    def last(self) -> Dict[str, Any]:
+        return self.requests[-1]
+
+
+def handler(stub: Stub):
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *args: Any) -> None:
+            pass
+
+        def send(self, status: int, body: Any, content_type: str = "application/json", headers: Dict[str, str] = {}) -> None:
+            data = body.encode() if isinstance(body, str) else json.dumps(body).encode()
+            self.send_response(status)
+            self.send_header("Content-Type", content_type)
+            self.send_header("Content-Length", str(len(data)))
+            for k, v in headers.items():
+                self.send_header(k, v)
+            self.end_headers()
+            self.wfile.write(data)
+
+        def problem(self, status: int, code: str, detail: str, headers: Dict[str, str] = {}) -> None:
+            body = {"type": f"{stub.base_url}/docs/#errors", "title": "Error", "status": status, "code": code, "detail": detail}
+            self.send(status, body, "application/problem+json", headers)
+
+        def do_GET(self) -> None:
+            parts = urlsplit(self.path)
+            query = {k: v[0] for k, v in parse_qs(parts.query).items()}
+            stub.requests.append({"path": parts.path, "query": query, "headers": dict(self.headers)})
+            auth = self.headers.get("Authorization")
+            if auth and auth != f"Bearer {GOOD_KEY}":
+                return self.problem(401, "invalid_key", "That API key is unknown or revoked.")
+            url = query.get("url", "")
+            if url == "html.test":
+                return self.send(502, "<html>Bad gateway</html>", "text/html")
+            if url in PROBLEMS:
+                status, code, detail, headers = PROBLEMS[url]
+                return self.problem(status, code, detail, headers)
+            if parts.path == "/api/v0/llms":
+                return self.send(200, result(url, query.get("mode", "llms")))
+            if parts.path == "/api/v0/check":
+                return self.send(200, CHECK)
+            self.problem(404, "not_found", "Not found.")
+
+    return Handler
+
+
+@pytest.fixture
+def stub(monkeypatch: pytest.MonkeyPatch):
+    """The stub server, with LLMS_INK_API pointing at it and LLMS_INK_KEY unset."""
+    s = Stub()
+    server = ThreadingHTTPServer(("127.0.0.1", 0), handler(s))
+    s.base_url = f"http://127.0.0.1:{server.server_address[1]}"
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    monkeypatch.setenv("LLMS_INK_API", s.base_url)
+    monkeypatch.delenv("LLMS_INK_KEY", raising=False)
+    yield s
+    server.shutdown()
+    server.server_close()

llms_ink-0.1.0/tests/test_cli.py

@@ -0,0 +1,43 @@
+import json
+import os
+import subprocess
+import sys
+
+from llms_ink.__main__ import main
+
+from .conftest import FULL, LLMS
+
+
+def test_prints_llms_txt(stub, capsys):
+    assert main(["llmstxt.org"]) == 0
+    assert capsys.readouterr().out == LLMS
+    assert stub.last["query"] == {"url": "llmstxt.org", "mode": "llms"}
+
+
+def test_full_and_fresh(stub, capsys):
+    assert main(["llmstxt.org", "--full", "--fresh"]) == 0
+    assert capsys.readouterr().out == FULL
+    assert stub.last["query"] == {"url": "llmstxt.org", "mode": "full", "fresh": "1"}
+
+
+def test_json(stub, capsys):
+    assert main(["llmstxt.org", "--json"]) == 0
+    assert json.loads(capsys.readouterr().out)["generatedAt"] == "2026-11-14T00:00:00.000Z"
+
+
+def test_errors_go_to_stderr_with_exit_1(stub, capsys):
+    assert main(["quota.test"]) == 1
+    out = capsys.readouterr()
+    assert out.out == ""
+    assert out.err == "llms-ink: Daily quota of new links used up. Try again in 3600s.\n"
+
+
+def test_runs_as_a_module(stub):
+    env = {**os.environ, "LLMS_INK_API": stub.base_url}
+    env.pop("LLMS_INK_KEY", None)
+    ok = subprocess.run([sys.executable, "-m", "llms_ink", "llmstxt.org"], env=env, capture_output=True, text=True)
+    assert (ok.returncode, ok.stdout) == (0, LLMS)
+    bad = subprocess.run([sys.executable, "-m", "llms_ink", "login.test"], env=env, capture_output=True, text=True)
+    assert (bad.returncode, bad.stderr) == (1, "llms-ink: That page needs a login.\n")
+    usage = subprocess.run([sys.executable, "-m", "llms_ink"], env=env, capture_output=True, text=True)
+    assert usage.returncode == 2

llms_ink-0.1.0/tests/test_client.py

@@ -0,0 +1,136 @@
+import pytest
+
+from llms_ink import (
+    Client,
+    InvalidKey,
+    LlmsInkError,
+    QuotaExceeded,
+    RateLimited,
+    Result,
+    __version__,
+)
+
+from .conftest import FULL, GOOD_KEY, LLMS
+
+
+def test_llms_returns_the_index_as_a_result(stub):
+    r = Client().llms("llmstxt.org")
+    assert isinstance(r, Result)
+    assert r.text == LLMS
+    assert (r.url, r.kind, r.source, r.mode, r.title) == ("https://llmstxt.org/", "site", "native", "llms", "llms.txt")
+    assert (r.tokens, r.pages, r.model, r.cached, r.generated_at) == (42, 1, None, True, "2026-11-14T00:00:00.000Z")
+    assert r.media is None
+    assert r.raw["generatedAt"] == r.generated_at
+    assert stub.last["path"] == "/api/v0/llms"
+    assert stub.last["query"] == {"url": "llmstxt.org", "mode": "llms"}
+    assert "Authorization" not in stub.last["headers"]
+    assert stub.last["headers"]["User-Agent"] == f"llms-ink-python/{__version__}"
+    assert stub.last["headers"]["Accept"] == "application/json"
+
+
+def test_full_mode_and_fresh(stub):
+    r = Client().llms("llmstxt.org", mode="full", fresh=True)
+    assert (r.mode, r.text) == ("full", FULL)
+    assert stub.last["query"] == {"url": "llmstxt.org", "mode": "full", "fresh": "1"}
+
+
+def test_bad_mode_fails_before_any_request(stub):
+    with pytest.raises(ValueError, match="mode"):
+        Client().llms("llmstxt.org", mode="index")
+    assert stub.requests == []
+
+
+def test_video_results_have_media(stub):
+    m = Client().llms("youtu.be/x").media
+    assert m is not None
+    assert (m.duration, m.has_video, m.has_audio, m.downloads, m.is_live) == (61.5, True, True, ["mp4", "m4a"], None)
+    assert m.transcript.source == "captions"
+    assert (m.transcript.segments[0].start, m.transcript.segments[0].text) == (0, "Hello")
+
+
+def test_check(stub):
+    c = Client().check("llmstxt.org")
+    assert stub.last["path"] == "/api/v0/check"
+    assert stub.last["query"] == {"url": "llmstxt.org"}
+    assert (c.found, c.status, c.title, c.sections, c.llms_full) == (True, 200, "llms.txt", 2, False)
+    assert (c.links.total, c.links.broken[0].url, c.links.broken[0].status) == (3, "https://llmstxt.org/gone", 404)
+    assert (c.issues[0].level, c.issues[0].code, c.issues[0].line) == ("error", "broken_link", 9)
+    assert (c.summary.errors, c.summary.warnings, c.checked_at) == (1, 0, "2026-11-14T00:00:00.000Z")
+
+
+def test_key_argument_is_sent_as_bearer(stub):
+    Client(GOOD_KEY).llms("llmstxt.org")
+    assert stub.last["headers"]["Authorization"] == f"Bearer {GOOD_KEY}"
+
+
+def test_key_comes_from_env_and_the_argument_wins(stub, monkeypatch):
+    monkeypatch.setenv("LLMS_INK_KEY", GOOD_KEY)
+    Client().llms("llmstxt.org")
+    assert stub.last["headers"]["Authorization"] == f"Bearer {GOOD_KEY}"
+    with pytest.raises(InvalidKey):
+        Client("lk_other").llms("llmstxt.org")
+    assert stub.last["headers"]["Authorization"] == "Bearer lk_other"
+
+
+def test_blank_env_key_is_anonymous(stub, monkeypatch):
+    monkeypatch.setenv("LLMS_INK_KEY", "  ")
+    Client().llms("llmstxt.org")
+    assert "Authorization" not in stub.last["headers"]
+
+
+def test_invalid_key_raises_and_never_falls_back(stub):
+    with pytest.raises(InvalidKey) as info:
+        Client("lk_revoked").llms("llmstxt.org")
+    assert (info.value.status, info.value.code) == (401, "invalid_key")
+    assert info.value.detail == "That API key is unknown or revoked."
+    assert len(stub.requests) == 1
+
+
+def test_quota_exceeded_has_retry_after(stub):
+    with pytest.raises(QuotaExceeded) as info:
+        Client().llms("quota.test")
+    assert (info.value.status, info.value.code, info.value.retry_after) == (429, "quota_exceeded", 3600)
+    assert isinstance(info.value, LlmsInkError)
+
+
+def test_rate_limited_has_retry_after(stub):
+    with pytest.raises(RateLimited) as info:
+        Client().llms("busy.test")
+    assert (info.value.code, info.value.retry_after) == ("rate_limited", 12)
+
+
+def test_other_problems_are_llms_ink_errors(stub):
+    with pytest.raises(LlmsInkError) as info:
+        Client().llms("login.test")
+    assert type(info.value) is LlmsInkError
+    assert (info.value.status, info.value.code, info.value.detail) == (422, "unprocessable", "That page needs a login.")
+    assert str(info.value) == "That page needs a login."
+
+
+def test_non_json_errors_keep_the_status(stub):
+    with pytest.raises(LlmsInkError) as info:
+        Client().llms("html.test")
+    assert (info.value.status, info.value.code) == (502, None)
+    assert "502" in info.value.detail
+
+
+def test_unreachable_api():
+    with pytest.raises(LlmsInkError, match="Couldn't reach http://127.0.0.1:9") as info:
+        Client(base_url="http://127.0.0.1:9", timeout=5).llms("llmstxt.org")
+    assert info.value.status is None
+
+
+def test_base_url_argument_and_trailing_slash(stub, monkeypatch):
+    monkeypatch.setenv("LLMS_INK_API", "http://127.0.0.1:9")
+    assert Client(base_url=stub.base_url + "/").llms("llmstxt.org").text == LLMS
+    monkeypatch.delenv("LLMS_INK_API")
+    assert Client().base_url == "https://llms.ink"
+
+
+def test_retry_after_parsing():
+    from llms_ink.client import _retry_after
+
+    assert _retry_after(None) is None
+    assert _retry_after("120") == 120
+    assert _retry_after("Wed, 21 Oct 2015 07:28:00 GMT") == 0
+    assert _retry_after("soon") is None

llms_ink-0.1.0/tests/test_live.py

@@ -0,0 +1,16 @@
+"""One real call to https://llms.ink. Skipped by default; run with: pytest -m live"""
+
+import pytest
+
+from llms_ink import Client
+
+pytestmark = pytest.mark.live
+
+
+def test_llmstxt_org(monkeypatch):
+    monkeypatch.delenv("LLMS_INK_API", raising=False)
+    r = Client().llms("llmstxt.org")
+    assert r.mode == "llms"
+    assert r.text.startswith("# ")
+    assert r.tokens > 0
+    assert r.url.startswith("https://llmstxt.org")

llms_ink-0.1.0/tests/test_loaders.py

@@ -0,0 +1,75 @@
+import pytest
+
+from llms_ink import Client, QuotaExceeded
+
+from .conftest import FULL, GOOD_KEY, LLMS
+
+METADATA = {
+    "url": "https://llmstxt.org/",
+    "title": "llms.txt",
+    "kind": "site",
+    "source": "native",
+    "mode": "full",
+    "tokens": 42,
+    "generated_at": "2026-11-14T00:00:00.000Z",
+}
+
+
+def test_langchain_loader(stub):
+    from langchain_core.document_loaders import BaseLoader
+    from langchain_core.documents import Document
+
+    from llms_ink.langchain import LlmsInkLoader
+
+    loader = LlmsInkLoader(["llmstxt.org", "youtu.be/x"], mode="full")
+    assert isinstance(loader, BaseLoader)
+    docs = loader.load()
+    assert [type(d) for d in docs] == [Document, Document]
+    assert docs[0].page_content == FULL
+    assert docs[0].metadata == METADATA
+    assert docs[1].metadata["kind"] == "video"
+    assert [r["query"]["url"] for r in stub.requests] == ["llmstxt.org", "youtu.be/x"]
+
+
+def test_langchain_loader_takes_one_url_a_key_and_a_client(stub):
+    from llms_ink.langchain import LlmsInkLoader
+
+    docs = list(LlmsInkLoader("llmstxt.org", key=GOOD_KEY).lazy_load())
+    assert docs[0].page_content == LLMS
+    assert docs[0].metadata["mode"] == "llms"
+    assert stub.last["headers"]["Authorization"] == f"Bearer {GOOD_KEY}"
+    client = Client()
+    assert LlmsInkLoader("x", client=client).client is client
+    with pytest.raises(ValueError):
+        LlmsInkLoader("x", mode="index")
+
+
+def test_langchain_loader_raises_api_errors(stub):
+    from llms_ink.langchain import LlmsInkLoader
+
+    with pytest.raises(QuotaExceeded):
+        LlmsInkLoader(["quota.test"]).load()
+
+
+def test_llama_index_reader(stub):
+    from llama_index.core import Document
+    from llama_index.core.readers.base import BaseReader
+
+    from llms_ink.llama_index import LlmsInkReader
+
+    reader = LlmsInkReader(mode="full")
+    assert isinstance(reader, BaseReader)
+    docs = reader.load_data(["llmstxt.org"])
+    assert [type(d) for d in docs] == [Document]
+    assert docs[0].text == FULL
+    assert docs[0].metadata == METADATA
+    assert [d.text for d in reader.load_data("llmstxt.org", mode="llms")] == [LLMS]
+    assert stub.last["query"]["mode"] == "llms"
+
+
+def test_llama_index_reader_key_and_fresh(stub):
+    from llms_ink.llama_index import LlmsInkReader
+
+    list(LlmsInkReader(fresh=True, key=GOOD_KEY).lazy_load_data(["llmstxt.org"]))
+    assert stub.last["query"] == {"url": "llmstxt.org", "mode": "llms", "fresh": "1"}
+    assert stub.last["headers"]["Authorization"] == f"Bearer {GOOD_KEY}"