llms-ink 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llms_ink/__init__.py +41 -0
- llms_ink/__main__.py +45 -0
- llms_ink/client.py +289 -0
- llms_ink/errors.py +56 -0
- llms_ink/langchain.py +44 -0
- llms_ink/llama_index.py +46 -0
- llms_ink/py.typed +0 -0
- llms_ink-0.1.0.dist-info/METADATA +134 -0
- llms_ink-0.1.0.dist-info/RECORD +11 -0
- llms_ink-0.1.0.dist-info/WHEEL +4 -0
- llms_ink-0.1.0.dist-info/licenses/LICENSE +21 -0
llms_ink/__init__.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Turn any link — site, docs section, page, GitHub repo, PDF or video — into llms.txt and llms-full.txt.
|
|
2
|
+
|
|
3
|
+
>>> from llms_ink import Client
|
|
4
|
+
>>> Client().llms("llmstxt.org").text
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
|
|
9
|
+
from .client import ( # noqa: E402
|
|
10
|
+
DEFAULT_BASE_URL,
|
|
11
|
+
BrokenLink,
|
|
12
|
+
CheckIssue,
|
|
13
|
+
CheckLinks,
|
|
14
|
+
CheckResult,
|
|
15
|
+
CheckSummary,
|
|
16
|
+
Client,
|
|
17
|
+
Media,
|
|
18
|
+
Result,
|
|
19
|
+
Segment,
|
|
20
|
+
Transcript,
|
|
21
|
+
)
|
|
22
|
+
from .errors import InvalidKey, LlmsInkError, QuotaExceeded, RateLimited # noqa: E402
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"DEFAULT_BASE_URL",
|
|
26
|
+
"BrokenLink",
|
|
27
|
+
"CheckIssue",
|
|
28
|
+
"CheckLinks",
|
|
29
|
+
"CheckResult",
|
|
30
|
+
"CheckSummary",
|
|
31
|
+
"Client",
|
|
32
|
+
"InvalidKey",
|
|
33
|
+
"LlmsInkError",
|
|
34
|
+
"Media",
|
|
35
|
+
"QuotaExceeded",
|
|
36
|
+
"RateLimited",
|
|
37
|
+
"Result",
|
|
38
|
+
"Segment",
|
|
39
|
+
"Transcript",
|
|
40
|
+
"__version__",
|
|
41
|
+
]
|
llms_ink/__main__.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""python -m llms_ink <url> [--full] [--fresh] [--json]: prints llms.txt (or llms-full.txt) for a link."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from typing import List, Optional
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
from .client import Client
|
|
12
|
+
from .errors import LlmsInkError, QuotaExceeded, RateLimited
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
16
|
+
parser = argparse.ArgumentParser(
|
|
17
|
+
prog="python -m llms_ink",
|
|
18
|
+
description="Print llms.txt for any link: a site, docs section, page, GitHub repo, PDF or video.",
|
|
19
|
+
epilog="Uses LLMS_INK_KEY as the API key when set (optional).",
|
|
20
|
+
)
|
|
21
|
+
parser.add_argument("url", help="any public link, e.g. llmstxt.org")
|
|
22
|
+
parser.add_argument("--full", action="store_true", help="llms-full.txt (the content) instead of the index")
|
|
23
|
+
parser.add_argument("--fresh", action="store_true", help="skip the cache and rebuild")
|
|
24
|
+
parser.add_argument("--json", action="store_true", help="the whole API result as JSON")
|
|
25
|
+
parser.add_argument("--version", action="version", version=f"llms-ink {__version__}")
|
|
26
|
+
args = parser.parse_args(argv)
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
result = Client().llms(args.url, mode="full" if args.full else "llms", fresh=args.fresh)
|
|
30
|
+
except LlmsInkError as err:
|
|
31
|
+
message = err.detail
|
|
32
|
+
if isinstance(err, (RateLimited, QuotaExceeded)) and err.retry_after is not None:
|
|
33
|
+
message += f" Try again in {err.retry_after}s."
|
|
34
|
+
print(f"llms-ink: {message}", file=sys.stderr)
|
|
35
|
+
return 1
|
|
36
|
+
|
|
37
|
+
if args.json:
|
|
38
|
+
print(json.dumps(result.raw, indent=2, ensure_ascii=False))
|
|
39
|
+
else:
|
|
40
|
+
sys.stdout.write(result.text if result.text.endswith("\n") else result.text + "\n")
|
|
41
|
+
return 0
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
if __name__ == "__main__":
|
|
45
|
+
sys.exit(main())
|
llms_ink/client.py
ADDED
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""The llms.ink JSON API (https://llms.ink/docs/) over the standard library's urllib."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import socket
|
|
8
|
+
import urllib.error
|
|
9
|
+
import urllib.parse
|
|
10
|
+
import urllib.request
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from email.utils import parsedate_to_datetime
|
|
14
|
+
from typing import Any, Dict, List, Optional
|
|
15
|
+
|
|
16
|
+
from . import __version__
|
|
17
|
+
from .errors import LlmsInkError, error_for
|
|
18
|
+
|
|
19
|
+
DEFAULT_BASE_URL = "https://llms.ink"
|
|
20
|
+
MODES = ("llms", "full")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class Segment:
|
|
25
|
+
"""A transcript line; ``start`` and ``end`` are seconds into the source video."""
|
|
26
|
+
|
|
27
|
+
start: float
|
|
28
|
+
end: float
|
|
29
|
+
text: str
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Transcript:
|
|
34
|
+
source: str
|
|
35
|
+
segments: List[Segment]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class Media:
|
|
40
|
+
"""Video results only: what ``/api/v0/frame`` and ``/api/v0/media`` can serve."""
|
|
41
|
+
|
|
42
|
+
duration: Optional[float]
|
|
43
|
+
has_video: bool
|
|
44
|
+
has_audio: bool
|
|
45
|
+
thumbnail: Optional[str]
|
|
46
|
+
downloads: List[str]
|
|
47
|
+
is_live: Optional[bool] = None
|
|
48
|
+
transcript: Optional[Transcript] = None
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def from_json(cls, d: Dict[str, Any]) -> "Media":
|
|
52
|
+
t = d.get("transcript")
|
|
53
|
+
return cls(
|
|
54
|
+
duration=d.get("duration"),
|
|
55
|
+
has_video=bool(d.get("hasVideo")),
|
|
56
|
+
has_audio=bool(d.get("hasAudio")),
|
|
57
|
+
thumbnail=d.get("thumbnail"),
|
|
58
|
+
downloads=list(d.get("downloads") or []),
|
|
59
|
+
is_live=d.get("isLive"),
|
|
60
|
+
transcript=Transcript(
|
|
61
|
+
source=t.get("source", ""),
|
|
62
|
+
segments=[Segment(s["start"], s["end"], s["text"]) for s in t.get("segments") or []],
|
|
63
|
+
)
|
|
64
|
+
if t
|
|
65
|
+
else None,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass
|
|
70
|
+
class Result:
|
|
71
|
+
"""``GET /api/v0/llms``: the llms.txt or llms-full.txt of a link and what it was built from.
|
|
72
|
+
|
|
73
|
+
``kind`` is ``site``, ``section``, ``page``, ``repo``, ``document`` or ``video``; ``source`` is ``native``
|
|
74
|
+
(published by the site) or ``generated`` (made by llms.ink). ``text`` is the Markdown file.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
url: str
|
|
78
|
+
kind: str
|
|
79
|
+
source: str
|
|
80
|
+
mode: str
|
|
81
|
+
title: str
|
|
82
|
+
text: str
|
|
83
|
+
tokens: int
|
|
84
|
+
pages: int
|
|
85
|
+
model: Optional[str]
|
|
86
|
+
cached: bool
|
|
87
|
+
generated_at: str
|
|
88
|
+
media: Optional[Media] = None
|
|
89
|
+
raw: Dict[str, Any] = field(default_factory=dict, repr=False, compare=False)
|
|
90
|
+
|
|
91
|
+
@classmethod
|
|
92
|
+
def from_json(cls, d: Dict[str, Any]) -> "Result":
|
|
93
|
+
return cls(
|
|
94
|
+
url=d.get("url", ""),
|
|
95
|
+
kind=d.get("kind", ""),
|
|
96
|
+
source=d.get("source", ""),
|
|
97
|
+
mode=d.get("mode", ""),
|
|
98
|
+
title=d.get("title", ""),
|
|
99
|
+
text=d.get("text", ""),
|
|
100
|
+
tokens=d.get("tokens", 0),
|
|
101
|
+
pages=d.get("pages", 0),
|
|
102
|
+
model=d.get("model"),
|
|
103
|
+
cached=bool(d.get("cached")),
|
|
104
|
+
generated_at=d.get("generatedAt", ""),
|
|
105
|
+
media=Media.from_json(d["media"]) if d.get("media") else None,
|
|
106
|
+
raw=d,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass
|
|
111
|
+
class CheckIssue:
|
|
112
|
+
"""``level`` is ``error``, ``warning`` or ``info``; ``code`` names the check (``missing_title``, ``broken_link``…)."""
|
|
113
|
+
|
|
114
|
+
level: str
|
|
115
|
+
code: str
|
|
116
|
+
line: Optional[int]
|
|
117
|
+
message: str
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass
|
|
121
|
+
class BrokenLink:
|
|
122
|
+
url: str
|
|
123
|
+
status: int # 0 when it didn't answer in time
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@dataclass
|
|
127
|
+
class CheckLinks:
|
|
128
|
+
total: int
|
|
129
|
+
checked: int
|
|
130
|
+
broken: List[BrokenLink]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass
|
|
134
|
+
class CheckSummary:
|
|
135
|
+
errors: int
|
|
136
|
+
warnings: int
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass
|
|
140
|
+
class CheckResult:
|
|
141
|
+
"""``GET /api/v0/check``: a published llms.txt checked against the format and its links."""
|
|
142
|
+
|
|
143
|
+
site: str
|
|
144
|
+
url: str
|
|
145
|
+
found: bool
|
|
146
|
+
status: int
|
|
147
|
+
title: Optional[str]
|
|
148
|
+
tokens: int
|
|
149
|
+
sections: int
|
|
150
|
+
links: CheckLinks
|
|
151
|
+
llms_full: bool
|
|
152
|
+
issues: List[CheckIssue]
|
|
153
|
+
summary: CheckSummary
|
|
154
|
+
checked_at: str
|
|
155
|
+
raw: Dict[str, Any] = field(default_factory=dict, repr=False, compare=False)
|
|
156
|
+
|
|
157
|
+
@classmethod
|
|
158
|
+
def from_json(cls, d: Dict[str, Any]) -> "CheckResult":
|
|
159
|
+
links = d.get("links") or {}
|
|
160
|
+
summary = d.get("summary") or {}
|
|
161
|
+
return cls(
|
|
162
|
+
site=d.get("site", ""),
|
|
163
|
+
url=d.get("url", ""),
|
|
164
|
+
found=bool(d.get("found")),
|
|
165
|
+
status=d.get("status", 0),
|
|
166
|
+
title=d.get("title"),
|
|
167
|
+
tokens=d.get("tokens", 0),
|
|
168
|
+
sections=d.get("sections", 0),
|
|
169
|
+
links=CheckLinks(
|
|
170
|
+
total=links.get("total", 0),
|
|
171
|
+
checked=links.get("checked", 0),
|
|
172
|
+
broken=[BrokenLink(b["url"], b["status"]) for b in links.get("broken") or []],
|
|
173
|
+
),
|
|
174
|
+
llms_full=bool(d.get("llmsFull")),
|
|
175
|
+
issues=[CheckIssue(i["level"], i["code"], i.get("line"), i["message"]) for i in d.get("issues") or []],
|
|
176
|
+
summary=CheckSummary(errors=summary.get("errors", 0), warnings=summary.get("warnings", 0)),
|
|
177
|
+
checked_at=d.get("checkedAt", ""),
|
|
178
|
+
raw=d,
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class Client:
|
|
183
|
+
"""A client for https://llms.ink.
|
|
184
|
+
|
|
185
|
+
``key`` is an optional API key (``lk_…``), sent as ``Authorization: Bearer``; it defaults to the
|
|
186
|
+
``LLMS_INK_KEY`` environment variable. Without one, requests are anonymous and share the free limits.
|
|
187
|
+
``base_url`` defaults to ``LLMS_INK_API`` or https://llms.ink. Building a big site or a video can take
|
|
188
|
+
minutes, hence the long default ``timeout`` (seconds).
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
def __init__(
|
|
192
|
+
self,
|
|
193
|
+
key: Optional[str] = None,
|
|
194
|
+
*,
|
|
195
|
+
base_url: Optional[str] = None,
|
|
196
|
+
timeout: float = 300.0,
|
|
197
|
+
) -> None:
|
|
198
|
+
key = key if key is not None else os.environ.get("LLMS_INK_KEY")
|
|
199
|
+
self.key = key.strip() if key and key.strip() else None
|
|
200
|
+
self.base_url = (base_url or os.environ.get("LLMS_INK_API") or DEFAULT_BASE_URL).rstrip("/")
|
|
201
|
+
self.timeout = timeout
|
|
202
|
+
|
|
203
|
+
def __repr__(self) -> str:
|
|
204
|
+
return f"Client(base_url={self.base_url!r}, key={'set' if self.key else None!r})"
|
|
205
|
+
|
|
206
|
+
def llms(self, url: str, mode: str = "llms", fresh: bool = False) -> Result:
|
|
207
|
+
"""llms.txt (``mode='llms'``, an index of links) or llms-full.txt (``mode='full'``, the content inlined)
|
|
208
|
+
for any public link. ``fresh=True`` skips the cache and rebuilds (counts against the daily quota)."""
|
|
209
|
+
if mode not in MODES:
|
|
210
|
+
raise ValueError(f"mode must be 'llms' (the index) or 'full' (llms-full.txt), not {mode!r}")
|
|
211
|
+
params = {"url": url, "mode": mode}
|
|
212
|
+
if fresh:
|
|
213
|
+
params["fresh"] = "1"
|
|
214
|
+
return Result.from_json(self._get("/api/v0/llms", params))
|
|
215
|
+
|
|
216
|
+
def check(self, url: str) -> CheckResult:
|
|
217
|
+
"""Checks a site's published llms.txt against the format and that its links answer."""
|
|
218
|
+
return CheckResult.from_json(self._get("/api/v0/check", {"url": url}))
|
|
219
|
+
|
|
220
|
+
def _get(self, path: str, params: Dict[str, str]) -> Dict[str, Any]:
|
|
221
|
+
headers = {"Accept": "application/json", "User-Agent": f"llms-ink-python/{__version__}"}
|
|
222
|
+
if self.key:
|
|
223
|
+
headers["Authorization"] = f"Bearer {self.key}"
|
|
224
|
+
request = urllib.request.Request(f"{self.base_url}{path}?{urllib.parse.urlencode(params)}", headers=headers)
|
|
225
|
+
try:
|
|
226
|
+
with urllib.request.urlopen(request, timeout=self.timeout) as res:
|
|
227
|
+
body = res.read()
|
|
228
|
+
except urllib.error.HTTPError as err:
|
|
229
|
+
raise _problem(err) from None
|
|
230
|
+
except (urllib.error.URLError, socket.timeout, TimeoutError, ConnectionError) as err:
|
|
231
|
+
reason = getattr(err, "reason", err)
|
|
232
|
+
raise LlmsInkError(f"Couldn't reach {self.base_url}: {reason}") from err
|
|
233
|
+
try:
|
|
234
|
+
return json.loads(body)
|
|
235
|
+
except ValueError as err:
|
|
236
|
+
raise LlmsInkError(f"{self.base_url}{path} answered with something other than JSON.") from err
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _problem(err: urllib.error.HTTPError) -> LlmsInkError:
|
|
240
|
+
"""An HTTP error response → the typed error for its problem ``code``."""
|
|
241
|
+
raw = err.read()
|
|
242
|
+
try:
|
|
243
|
+
data = json.loads(raw)
|
|
244
|
+
except ValueError:
|
|
245
|
+
data = None
|
|
246
|
+
if not isinstance(data, dict):
|
|
247
|
+
data = {}
|
|
248
|
+
status = data.get("status") if isinstance(data.get("status"), int) else err.code
|
|
249
|
+
detail = data.get("detail") or data.get("title") or f"HTTP {err.code} {err.reason or 'error'}"
|
|
250
|
+
code = data.get("code")
|
|
251
|
+
return error_for(code)(
|
|
252
|
+
detail,
|
|
253
|
+
status=status,
|
|
254
|
+
code=code,
|
|
255
|
+
title=data.get("title"),
|
|
256
|
+
errors=data.get("errors"),
|
|
257
|
+
retry_after=_retry_after(err.headers.get("Retry-After") if err.headers else None),
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _retry_after(value: Optional[str]) -> Optional[int]:
|
|
262
|
+
"""``Retry-After`` as seconds: delta-seconds, or an HTTP date turned into seconds from now."""
|
|
263
|
+
if not value:
|
|
264
|
+
return None
|
|
265
|
+
value = value.strip()
|
|
266
|
+
if value.isdigit():
|
|
267
|
+
return int(value)
|
|
268
|
+
try:
|
|
269
|
+
when = parsedate_to_datetime(value)
|
|
270
|
+
except (TypeError, ValueError):
|
|
271
|
+
return None
|
|
272
|
+
if when is None:
|
|
273
|
+
return None
|
|
274
|
+
if when.tzinfo is None:
|
|
275
|
+
when = when.replace(tzinfo=timezone.utc)
|
|
276
|
+
return max(0, int((when - datetime.now(timezone.utc)).total_seconds()))
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _metadata(result: Result) -> Dict[str, Any]:
|
|
280
|
+
"""Document metadata for the LangChain and LlamaIndex loaders."""
|
|
281
|
+
return {
|
|
282
|
+
"url": result.url,
|
|
283
|
+
"title": result.title,
|
|
284
|
+
"kind": result.kind,
|
|
285
|
+
"source": result.source,
|
|
286
|
+
"mode": result.mode,
|
|
287
|
+
"tokens": result.tokens,
|
|
288
|
+
"generated_at": result.generated_at,
|
|
289
|
+
}
|
llms_ink/errors.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Typed errors from the API's RFC 9457 problem details (codes at https://llms.ink/docs/#errors)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Dict, List, Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class LlmsInkError(Exception):
|
|
9
|
+
"""An API error, or llms.ink couldn't be reached (then ``status`` and ``code`` are ``None``).
|
|
10
|
+
|
|
11
|
+
``code`` is the problem's machine-readable code (``invalid_url``, ``unprocessable``, ``upstream_error``…),
|
|
12
|
+
``detail`` the human-readable message, ``errors`` the per-parameter issues of ``invalid_request``.
|
|
13
|
+
``retry_after`` is the ``Retry-After`` header in seconds, when the response had one.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
detail: str,
|
|
19
|
+
*,
|
|
20
|
+
status: Optional[int] = None,
|
|
21
|
+
code: Optional[str] = None,
|
|
22
|
+
title: Optional[str] = None,
|
|
23
|
+
errors: Optional[List[Dict[str, Any]]] = None,
|
|
24
|
+
retry_after: Optional[int] = None,
|
|
25
|
+
) -> None:
|
|
26
|
+
super().__init__(detail)
|
|
27
|
+
self.detail = detail
|
|
28
|
+
self.status = status
|
|
29
|
+
self.code = code
|
|
30
|
+
self.title = title
|
|
31
|
+
self.errors = errors or []
|
|
32
|
+
self.retry_after = retry_after
|
|
33
|
+
|
|
34
|
+
def __repr__(self) -> str:
|
|
35
|
+
return f"{type(self).__name__}(status={self.status!r}, code={self.code!r}, detail={self.detail!r})"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class InvalidKey(LlmsInkError):
|
|
39
|
+
"""401 ``invalid_key``: the API key is unknown or revoked. Requests never fall back to anonymous."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class RateLimited(LlmsInkError):
|
|
43
|
+
"""429 ``rate_limited``: too many requests this minute; wait ``retry_after`` seconds."""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class QuotaExceeded(LlmsInkError):
|
|
47
|
+
"""429 ``quota_exceeded``: the daily quota of new links is used up; ``retry_after`` is the seconds until it
|
|
48
|
+
resets (midnight UTC). Cached links still work."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
_BY_CODE = {"invalid_key": InvalidKey, "rate_limited": RateLimited, "quota_exceeded": QuotaExceeded}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def error_for(code: Optional[str]) -> type:
|
|
55
|
+
"""The exception class for a problem ``code``."""
|
|
56
|
+
return _BY_CODE.get(code or "", LlmsInkError)
|
llms_ink/langchain.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""LangChain document loader: ``pip install 'llms-ink[langchain]'``.
|
|
2
|
+
|
|
3
|
+
>>> from llms_ink.langchain import LlmsInkLoader
|
|
4
|
+
>>> docs = LlmsInkLoader(["docs.astro.build/en/guides/"], mode="full").load()
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Iterator, Optional, Sequence, Union
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
from langchain_core.document_loaders import BaseLoader
|
|
13
|
+
from langchain_core.documents import Document
|
|
14
|
+
except ImportError as err: # pragma: no cover
|
|
15
|
+
raise ImportError("LlmsInkLoader needs langchain-core: pip install 'llms-ink[langchain]'") from err
|
|
16
|
+
|
|
17
|
+
from .client import MODES, Client, _metadata
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class LlmsInkLoader(BaseLoader):
|
|
21
|
+
"""One Document per link: llms.txt (``mode='llms'``) or llms-full.txt (``mode='full'``) as
|
|
22
|
+
``page_content``, with metadata ``url``, ``title``, ``kind``, ``source``, ``mode``, ``tokens`` and
|
|
23
|
+
``generated_at``. ``key`` defaults to ``LLMS_INK_KEY``; pass ``client`` to reuse one."""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
urls: Union[str, Sequence[str]],
|
|
28
|
+
mode: str = "llms",
|
|
29
|
+
*,
|
|
30
|
+
fresh: bool = False,
|
|
31
|
+
key: Optional[str] = None,
|
|
32
|
+
client: Optional[Client] = None,
|
|
33
|
+
) -> None:
|
|
34
|
+
if mode not in MODES:
|
|
35
|
+
raise ValueError(f"mode must be 'llms' or 'full', not {mode!r}")
|
|
36
|
+
self.urls = [urls] if isinstance(urls, str) else list(urls)
|
|
37
|
+
self.mode = mode
|
|
38
|
+
self.fresh = fresh
|
|
39
|
+
self.client = client or Client(key)
|
|
40
|
+
|
|
41
|
+
def lazy_load(self) -> Iterator[Document]:
|
|
42
|
+
for url in self.urls:
|
|
43
|
+
result = self.client.llms(url, mode=self.mode, fresh=self.fresh)
|
|
44
|
+
yield Document(page_content=result.text, metadata=_metadata(result))
|
llms_ink/llama_index.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""LlamaIndex reader: ``pip install 'llms-ink[llama-index]'``.
|
|
2
|
+
|
|
3
|
+
>>> from llms_ink.llama_index import LlmsInkReader
|
|
4
|
+
>>> docs = LlmsInkReader(mode="full").load_data(["docs.astro.build/en/guides/"])
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Iterator, List, Optional, Union
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
from llama_index.core import Document
|
|
13
|
+
from llama_index.core.readers.base import BaseReader
|
|
14
|
+
except ImportError as err: # pragma: no cover
|
|
15
|
+
raise ImportError("LlmsInkReader needs llama-index-core: pip install 'llms-ink[llama-index]'") from err
|
|
16
|
+
|
|
17
|
+
from .client import MODES, Client, _metadata
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class LlmsInkReader(BaseReader):
|
|
21
|
+
"""One Document per link: llms.txt (``mode='llms'``) or llms-full.txt (``mode='full'``) as text, with
|
|
22
|
+
metadata ``url``, ``title``, ``kind``, ``source``, ``mode``, ``tokens`` and ``generated_at``.
|
|
23
|
+
``key`` defaults to ``LLMS_INK_KEY``; pass ``client`` to reuse one."""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
mode: str = "llms",
|
|
28
|
+
*,
|
|
29
|
+
fresh: bool = False,
|
|
30
|
+
key: Optional[str] = None,
|
|
31
|
+
client: Optional[Client] = None,
|
|
32
|
+
) -> None:
|
|
33
|
+
if mode not in MODES:
|
|
34
|
+
raise ValueError(f"mode must be 'llms' or 'full', not {mode!r}")
|
|
35
|
+
self.mode = mode
|
|
36
|
+
self.fresh = fresh
|
|
37
|
+
self.client = client or Client(key)
|
|
38
|
+
|
|
39
|
+
def lazy_load_data(self, urls: Union[str, List[str]], mode: Optional[str] = None) -> Iterator[Document]:
|
|
40
|
+
"""Documents for ``urls``; ``mode`` overrides the reader's for this call."""
|
|
41
|
+
for url in [urls] if isinstance(urls, str) else urls:
|
|
42
|
+
result = self.client.llms(url, mode=mode or self.mode, fresh=self.fresh)
|
|
43
|
+
yield Document(text=result.text, metadata=_metadata(result))
|
|
44
|
+
|
|
45
|
+
def load_data(self, urls: Union[str, List[str]], mode: Optional[str] = None) -> List[Document]:
|
|
46
|
+
return list(self.lazy_load_data(urls, mode))
|
llms_ink/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: llms-ink
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Turn any link — site, docs section, page, GitHub repo, PDF or video — into llms.txt and llms-full.txt. Python client for llms.ink, with LangChain and LlamaIndex loaders.
|
|
5
|
+
Project-URL: Homepage, https://llms.ink
|
|
6
|
+
Project-URL: Documentation, https://llms.ink/docs/
|
|
7
|
+
Author: Lucas Espindola
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: ai,context,docs,langchain,llama-index,llm,llms-full.txt,llms.txt,markdown,rag
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Provides-Extra: langchain
|
|
21
|
+
Requires-Dist: langchain-core>=0.2; extra == 'langchain'
|
|
22
|
+
Provides-Extra: llama-index
|
|
23
|
+
Requires-Dist: llama-index-core>=0.10; extra == 'llama-index'
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# llms-ink
|
|
27
|
+
|
|
28
|
+
Turn any public link — a site, docs section, page, GitHub repo, PDF or video — into
|
|
29
|
+
[llms.txt](https://llmstxt.org) or llms-full.txt, from Python. A client for [llms.ink](https://llms.ink)
|
|
30
|
+
with LangChain and LlamaIndex loaders. No required dependencies.
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
pip install llms-ink
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Use
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from llms_ink import Client
|
|
40
|
+
|
|
41
|
+
ink = Client()
|
|
42
|
+
index = ink.llms("llmstxt.org") # llms.txt: an index of links
|
|
43
|
+
docs = ink.llms("docs.astro.build/en/guides/", mode="full") # llms-full.txt: the content
|
|
44
|
+
|
|
45
|
+
print(docs.title, docs.kind, docs.tokens)
|
|
46
|
+
print(docs.text)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
`llms()` returns a `Result`: `url`, `kind` (`site`, `section`, `page`, `repo`, `document`, `video`),
|
|
50
|
+
`source` (`native` = published by the site, `generated` = made by llms.ink), `mode`, `title`, `text`,
|
|
51
|
+
`tokens`, `pages`, `model`, `cached`, `generated_at`, and `media` for videos (duration, downloads,
|
|
52
|
+
transcript segments). `fresh=True` skips the cache and rebuilds.
|
|
53
|
+
|
|
54
|
+
Check a site's published llms.txt:
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
report = ink.check("llmstxt.org")
|
|
58
|
+
print(report.found, report.summary.errors, [i.message for i in report.issues])
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## API key (optional)
|
|
62
|
+
|
|
63
|
+
Without a key you share the free anonymous limits (10 requests/minute, 60 new links a day per IP;
|
|
64
|
+
cached links are free). A key gets its own daily quota:
|
|
65
|
+
|
|
66
|
+
```sh
|
|
67
|
+
export LLMS_INK_KEY=lk_...
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
or `Client(key="lk_...")`. It is sent as `Authorization: Bearer`. A bad key raises `InvalidKey`; it never
|
|
71
|
+
falls back to anonymous. [Ask for a key](https://lucasespindola.com/message?text=I%27d+like+an+llms.ink+API+key).
|
|
72
|
+
|
|
73
|
+
## Errors
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from llms_ink import Client, LlmsInkError, QuotaExceeded, RateLimited
|
|
77
|
+
|
|
78
|
+
try:
|
|
79
|
+
Client().llms("example.com")
|
|
80
|
+
except (RateLimited, QuotaExceeded) as err:
|
|
81
|
+
print(f"wait {err.retry_after}s")
|
|
82
|
+
except LlmsInkError as err:
|
|
83
|
+
print(err.status, err.code, err.detail)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
`LlmsInkError` carries the API's `status`, `code` (see the [error table](https://llms.ink/docs/#errors)),
|
|
87
|
+
`detail` and `errors`. `InvalidKey`, `RateLimited` and `QuotaExceeded` are subclasses.
|
|
88
|
+
|
|
89
|
+
## LangChain
|
|
90
|
+
|
|
91
|
+
```sh
|
|
92
|
+
pip install 'llms-ink[langchain]'
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from llms_ink.langchain import LlmsInkLoader
|
|
97
|
+
|
|
98
|
+
docs = LlmsInkLoader(["llmstxt.org", "github.com/AnswerDotAI/llms-txt"], mode="full").load()
|
|
99
|
+
docs[0].metadata # {'url', 'title', 'kind', 'source', 'mode', 'tokens', 'generated_at'}
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## LlamaIndex
|
|
103
|
+
|
|
104
|
+
```sh
|
|
105
|
+
pip install 'llms-ink[llama-index]'
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from llama_index.core import VectorStoreIndex
|
|
110
|
+
from llms_ink.llama_index import LlmsInkReader
|
|
111
|
+
|
|
112
|
+
docs = LlmsInkReader(mode="full").load_data(["docs.astro.build/en/guides/"])
|
|
113
|
+
index = VectorStoreIndex.from_documents(docs)
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Command line
|
|
117
|
+
|
|
118
|
+
```sh
|
|
119
|
+
python -m llms_ink llmstxt.org # llms.txt
|
|
120
|
+
python -m llms_ink llmstxt.org --full # llms-full.txt
|
|
121
|
+
python -m llms_ink llmstxt.org --json # the whole result
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Development
|
|
125
|
+
|
|
126
|
+
```sh
|
|
127
|
+
python3 -m venv .venv && .venv/bin/pip install -e '.[langchain,llama-index]' pytest
|
|
128
|
+
.venv/bin/pytest # against a local stub server
|
|
129
|
+
.venv/bin/pytest -m live # one real call to https://llms.ink
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`LLMS_INK_API` points the client at another API base URL.
|
|
133
|
+
|
|
134
|
+
MIT licensed.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
llms_ink/__init__.py,sha256=RjdpYlU81hIDdkCb1nmDXMUEEqDuhvk40xu23VhynBk,814
|
|
2
|
+
llms_ink/__main__.py,sha256=dQa-hpBAGk_wPt0aD3_fZ2oXee9Fhy71fPgw4c1uIak,1778
|
|
3
|
+
llms_ink/client.py,sha256=6hC-X9tVe3K4og-ZITerGTOXx7vFcrkX8GBd5DVmsh8,9442
|
|
4
|
+
llms_ink/errors.py,sha256=Pqr4uZ8eW04P_DfC42pxLSSaX_Z2WdLhX4GX6fDXX6Q,1998
|
|
5
|
+
llms_ink/langchain.py,sha256=RJN41PWBQOaLA9uePF0o-vV561vLlCWJvAeRtaiFNak,1679
|
|
6
|
+
llms_ink/llama_index.py,sha256=PzeBKtGm3PUluXLSk0yNWL0x8YK9O2wn7ONBpSBqXag,1880
|
|
7
|
+
llms_ink/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
llms_ink-0.1.0.dist-info/METADATA,sha256=Nm--sH8uC_tNNVYyAtPpME0XFETJBiVIahx7lJdEP6I,4285
|
|
9
|
+
llms_ink-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
10
|
+
llms_ink-0.1.0.dist-info/licenses/LICENSE,sha256=FmG9pZzsJIZAObaW9nC7RL6Cx42IZ4yqJ7XH58Lj-w0,1072
|
|
11
|
+
llms_ink-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lucas Espindola
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|