ai-terms 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,44 @@
1
+ Metadata-Version: 2.4
2
+ Name: ai-terms
3
+ Version: 0.1.0
4
+ Summary: Read AI Terms files (/.well-known/ai-terms): what a site allows AI to do with its pages, per use, and at what price. One module, standard library only.
5
+ Project-URL: Homepage, https://aipaygen.com/ai-terms
6
+ Project-URL: Source, https://github.com/Damien829/pay-per-crawl/tree/main/readers/python
7
+ Author: Damien Ovalles (AIPayGen)
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Keywords: ai,ai-terms,aipref,crawler,licensing,robots,x402
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Requires-Python: >=3.8
15
+ Description-Content-Type: text/markdown
16
+
17
+ # AI Terms reader for Python
18
+
19
+ One module, standard library only, no AIPayGen dependency, Python 3.8 or later. MIT. Copy `ai_terms.py` into your project.
20
+
21
+ [AI Terms](https://aipaygen.com/ai-terms) is a JSON file a site publishes at `/.well-known/ai-terms` that says, for each use of its content by AI (search, answer, training, agent), whether that use is allowed, allowed for a price, or not allowed. It is a draft ([draft-ovalles-ai-terms](https://datatracker.ietf.org/doc/draft-ovalles-ai-terms/)), not a standard.
22
+
23
+ ```python
24
+ import ai_terms
25
+
26
+ terms = ai_terms.fetch("https://example.com") # None when the site publishes no terms
27
+ if terms:
28
+ rule = ai_terms.resolve(terms, "agent", "/blog/post")
29
+ if not ai_terms.allowed_within(rule, max_price_usd=0.01):
30
+ raise SystemExit("this site does not allow that use at my price")
31
+ # and declare what you are doing on every request:
32
+ headers = {"AI-Purpose": "agent"}
33
+ ```
34
+
35
+ - `fetch(site)` reads `/.well-known/ai-terms` over https, capped at 100 KB.
36
+ - `resolve(terms, use, path)` returns the rule that applies (`access`, and `price` / `attribution` when set).
37
+ - `allowed_within(rule, max_price_usd)` is the check a careful client makes before reading or paying.
38
+ - `validate(doc)` returns `(errors, warnings)`.
39
+
40
+ From a checkout of the [repository](https://github.com/Damien829/pay-per-crawl), run `python3 readers/python/test_vectors.py` to check it against the published [test vectors](https://github.com/Damien829/pay-per-crawl/blob/main/spec/ai-terms-vectors.json): 58 of 58 pass.
41
+
42
+ This is a second implementation of the format, in a different language from the reference one (TypeScript, in the AIPayGen gateway). It has the same author, so it is not yet the independent implementation a standard needs. If you write one, open a pull request and it will be listed here.
43
+
44
+ License: MIT.
@@ -0,0 +1,5 @@
1
+ ai_terms.py,sha256=KWQVlzPIPyMLXCA1hKxcQYYJZYRPuMjhyaLJAMPu5kU,17201
2
+ ai_terms-0.1.0.dist-info/METADATA,sha256=YWsibFqnLma_f3vcNW8gWbXltIuG4eaSuS_4v03FG8o,2539
3
+ ai_terms-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
4
+ ai_terms-0.1.0.dist-info/licenses/LICENSE,sha256=Han7O1lhD7scHuN8krJigmo75dUa48DUkHlQep7Dv8Q,1082
5
+ ai_terms-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.27.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Damien Ovalles (AIPayGen)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
ai_terms.py ADDED
@@ -0,0 +1,389 @@
1
+ """AI Terms 0.1 reader for Python. Standard library only, one file, no AIPayGen dependency.
2
+
3
+ AI Terms (https://aipaygen.com/ai-terms, draft-ovalles-ai-terms) is a JSON file at
4
+ /.well-known/ai-terms where a site says, per use, what AI may do with its pages:
5
+ allow, paid (with a price) or deny.
6
+
7
+ import ai_terms
8
+
9
+ terms = ai_terms.fetch("https://example.com") # None if the site publishes none
10
+ if terms:
11
+ rule = ai_terms.resolve(terms, "agent", "/blog/post")
12
+ if rule["access"] == "deny":
13
+ ... # do not use the page for this purpose
14
+ elif rule["access"] == "paid":
15
+ ... # pay rule["price"] through terms["payment"] first (x402), within your own cap
16
+
17
+ # Always tell the site what you are doing:
18
+ headers = {"AI-Purpose": "agent"}
19
+
20
+ This file was written from the spec text and checked against the published test vectors
21
+ (run `python3 test_vectors.py`). License: MIT.
22
+ """
23
+ import json
24
+ import re
25
+ import urllib.request
26
+ from datetime import date
27
+ from urllib.parse import urlsplit
28
+
29
+ VERSION = "0.1" # the AI Terms format version this reader implements
30
+ __version__ = "0.1.0" # this package's version (PyPI "ai-terms", matching npm "ai-terms")
31
+ USES = ("search", "answer", "training", "agent")
32
+ ACCESS = ("allow", "paid", "deny")
33
+ MAX_BYTES = 100_000
34
+
35
+
36
+ def parse(text):
37
+ """Returns the terms as a dict, or None when the text is not an AI Terms file."""
38
+ try:
39
+ doc = json.loads(text)
40
+ except (ValueError, TypeError):
41
+ return None
42
+ if isinstance(doc, dict) and isinstance(doc.get("ai_terms"), str) and isinstance(doc.get("uses"), dict):
43
+ return doc
44
+ return None
45
+
46
+
47
+ class _NoRedirect(urllib.request.HTTPRedirectHandler):
48
+ """Refuses every redirect: a 3xx answer is an error, so fetch() returns None (urllib follows them by default)."""
49
+
50
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
51
+ return None
52
+
53
+
54
+ _OPENER = urllib.request.build_opener(_NoRedirect)
55
+
56
+
57
+ def fetch(site, timeout=5.0, user_agent="ai-terms-python/0.1", purpose="agent"):
58
+ """Reads a site's terms from /.well-known/ai-terms. Returns None when there are none.
59
+
60
+ Only https is read, redirects are not followed (a 3xx answer gives None), and the body is capped.
61
+ """
62
+ parts = urlsplit(site if "://" in site else "https://" + site)
63
+ if parts.scheme != "https" or not parts.netloc:
64
+ return None
65
+ req = urllib.request.Request(
66
+ "https://%s/.well-known/ai-terms" % parts.netloc,
67
+ headers={"User-Agent": user_agent, "Accept": "application/json", "AI-Purpose": purpose},
68
+ )
69
+ try:
70
+ with _OPENER.open(req, timeout=timeout) as res:
71
+ if res.status != 200:
72
+ return None
73
+ return parse(res.read(MAX_BYTES).decode("utf-8", "replace"))
74
+ except Exception:
75
+ return None
76
+
77
+
78
+ GLOB_MAX_LEN, GLOB_MAX_STARS = 512, 16
79
+ _UNRESERVED = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~")
80
+ _PATH_CHARS = _UNRESERVED | set("!$&'()*+,;=:@/")
81
+ _HEX = "0123456789abcdefABCDEF"
82
+
83
+
84
+ def normalize_path(s):
85
+ """A path or pattern in one comparable form (RFC 3986 section 6.2.2): hex digits upper-case, encoded
86
+ unreserved characters decoded, anything outside the path set percent-encoded as UTF-8."""
87
+ out, i = [], 0
88
+ while i < len(s):
89
+ c = s[i]
90
+ if c == "%" and i + 2 < len(s) and s[i + 1] in _HEX and s[i + 2] in _HEX:
91
+ ch = chr(int(s[i + 1:i + 3], 16))
92
+ out.append(ch if ch in _UNRESERVED else "%" + s[i + 1:i + 3].upper())
93
+ i += 3
94
+ continue
95
+ if c in _PATH_CHARS:
96
+ out.append(c)
97
+ else:
98
+ out.extend("%%%02X" % b for b in c.encode("utf-8"))
99
+ i += 1
100
+ return "".join(out)
101
+
102
+
103
+ def _segment_match(pat, seg):
104
+ pieces = pat.split("*")
105
+ if len(pieces) == 1:
106
+ return pat == seg
107
+ first, last = pieces[0], pieces[-1]
108
+ if len(first) + len(last) > len(seg) or not seg.startswith(first) or not seg.endswith(last):
109
+ return False
110
+ at, end = len(first), len(seg) - len(last)
111
+ for piece in pieces[1:-1]:
112
+ if not piece:
113
+ continue
114
+ found = seg.find(piece, at)
115
+ if found < 0 or found + len(piece) > end:
116
+ return False
117
+ at = found + len(piece)
118
+ return True
119
+
120
+
121
+ def _matches(pattern, path):
122
+ # "*" matches any characters except "/"; the pattern must match the whole path. Linear: segment by segment,
123
+ # leftmost placement of each piece (never a backtracking regular expression).
124
+ if len(pattern) > GLOB_MAX_LEN or pattern.count("*") > GLOB_MAX_STARS:
125
+ return False
126
+ ps, ss = pattern.split("/"), path.split("/")
127
+ return len(ps) == len(ss) and all(_segment_match(p, q) for p, q in zip(ps, ss))
128
+
129
+
130
+ def _usable(rule):
131
+ return isinstance(rule, dict) and rule.get("access") in ("allow", "paid", "deny")
132
+
133
+
134
+ def resolve(terms, use, path="/"):
135
+ """The terms that apply to one use on one path, as a dict with at least "access".
136
+
137
+ Each use on its own: among the `paths` entries that match the path and list the use (as an object with a
138
+ valid access), the one with the most literal characters wins, the first on a tie; else the top-level
139
+ terms for the use; else `default`. Path and patterns are compared normalized (normalize_path). `path` is
140
+ the request path as a server resolves it: dot-segments already removed, no query.
141
+ """
142
+ if use not in USES:
143
+ raise ValueError("use must be one of %s" % ", ".join(USES))
144
+ best, best_len, p0 = None, -1, normalize_path(path)
145
+ paths = terms.get("paths")
146
+ for entry in paths if isinstance(paths, list) else []:
147
+ if not isinstance(entry, dict) or not isinstance(entry.get("match"), str) or not isinstance(entry.get("uses"), dict):
148
+ continue
149
+ rule = entry["uses"].get(use)
150
+ if not _usable(rule):
151
+ continue
152
+ m = normalize_path(entry["match"])
153
+ literal = len(m.replace("*", ""))
154
+ if _matches(m, p0) and literal > best_len:
155
+ best, best_len = rule, literal
156
+ if best is not None:
157
+ return best
158
+ uses = terms.get("uses")
159
+ rule = uses.get(use) if isinstance(uses, dict) else None
160
+ if _usable(rule):
161
+ return rule
162
+ d = terms.get("default")
163
+ return {"access": d if d in ("allow", "paid", "deny") else "allow"}
164
+
165
+
166
+ def content_usage(terms, path="/"):
167
+ """The IETF aipref Content-Usage value these terms imply for a path, e.g. "train-ai=n", or "" for none.
168
+
169
+ Mapping (draft-ovalles-ai-terms-03): train-ai is y when training is allowed, n when it is paid or denied;
170
+ ai-use is y when answer and agent are both allowed, n when neither is, otherwise not stated; search is y
171
+ when search and training are both allowed (aipref Search also allows training inside search), otherwise
172
+ not stated. A report_only document states nothing.
173
+ """
174
+ if terms.get("report_only") is True:
175
+ return ""
176
+ allowed = lambda use: resolve(terms, use, path)["access"] == "allow"
177
+ out = ["train-ai=" + ("y" if allowed("training") else "n")]
178
+ answer, agent = allowed("answer"), allowed("agent")
179
+ if answer and agent:
180
+ out.append("ai-use=y")
181
+ elif not answer and not agent:
182
+ out.append("ai-use=n")
183
+ if allowed("search") and allowed("training"):
184
+ out.append("search=y")
185
+ return ", ".join(out)
186
+
187
+
188
+ FREE_MAX_REQUESTS = 1000000000
189
+
190
+
191
+ def free_allowance(rule):
192
+ """The free allowance on a paid or denied use, as (requests, "hour" | "day"), or None.
193
+
194
+ A verified client may make that many requests for the use in each UTC hour or day before the
195
+ use's access applies. Unverified clients get none. Attribution still applies to free reads.
196
+ """
197
+ free = rule.get("free") if isinstance(rule, dict) else None
198
+ if not isinstance(free, dict) or rule.get("access") == "allow":
199
+ return None
200
+ n, per = free.get("requests"), free.get("per")
201
+ if isinstance(n, bool) or not isinstance(n, int) or not 1 <= n <= FREE_MAX_REQUESTS or per not in ("hour", "day"):
202
+ return None
203
+ return n, per
204
+
205
+
206
+ def allowed_within(rule, max_price_usd):
207
+ """True when a client may proceed: access is allow, or paid at or under the client's cap.
208
+
209
+ Clients MUST cap what they pay: a file can name any price.
210
+ """
211
+ if rule.get("access") == "allow":
212
+ return True
213
+ if rule.get("access") != "paid":
214
+ return False
215
+ try:
216
+ return float(rule["price"]["amount"]) <= float(max_price_usd)
217
+ except (KeyError, TypeError, ValueError):
218
+ return False
219
+
220
+
221
+ def validate(doc):
222
+ """Checks a parsed file against AI Terms 0.1. Returns (errors, warnings); valid when errors is empty."""
223
+ errors, warnings = [], []
224
+ if not isinstance(doc, dict):
225
+ return ["The file must be a JSON object."], warnings
226
+
227
+ if not isinstance(doc.get("ai_terms"), str):
228
+ errors.append('"ai_terms" is required: the spec version, "0.1".')
229
+ elif doc["ai_terms"] != VERSION:
230
+ warnings.append('"ai_terms" is "%s"; this reader knows %s.' % (doc["ai_terms"], VERSION))
231
+
232
+ site = doc.get("site")
233
+ if not isinstance(site, str):
234
+ errors.append('"site" is required: the origin the terms apply to.')
235
+ else:
236
+ parts = urlsplit(site)
237
+ if not parts.scheme or not parts.netloc:
238
+ errors.append('"site" must be a full URL, like "https://example.com".')
239
+ elif parts.scheme not in ("https", "http"):
240
+ errors.append('"site" must be a web origin, like "https://example.com".')
241
+ elif parts.scheme != "https":
242
+ warnings.append('"site" should be an https:// origin.')
243
+
244
+ updated = doc.get("updated")
245
+ ok_date = isinstance(updated, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", updated) is not None
246
+ if ok_date:
247
+ try:
248
+ date.fromisoformat(updated)
249
+ except ValueError:
250
+ ok_date = False
251
+ if not ok_date:
252
+ errors.append('"updated" is required: the date of the last change, as YYYY-MM-DD.')
253
+
254
+ if doc.get("default") not in ACCESS:
255
+ errors.append('"default" is required: "allow", "paid" or "deny".')
256
+
257
+ def check_use(rule, where):
258
+ """Returns True when the use is paid."""
259
+ if not isinstance(rule, dict):
260
+ errors.append('%s must be an object with "access".' % where)
261
+ return False
262
+ if rule.get("access") not in ACCESS:
263
+ errors.append('%s.access must be "allow", "paid" or "deny".' % where)
264
+ return False
265
+ if "attribution" in rule and rule["attribution"] not in ("required", "requested", "none"):
266
+ errors.append('%s.attribution must be "required", "requested" or "none".' % where)
267
+ if "free" in rule:
268
+ free = rule["free"]
269
+ if not isinstance(free, dict):
270
+ errors.append('%s.free must be an object: {"requests": 100, "per": "day"}.' % where)
271
+ else:
272
+ n = free.get("requests")
273
+ if isinstance(n, bool) or not isinstance(n, int) or not 1 <= n <= FREE_MAX_REQUESTS:
274
+ errors.append("%s.free.requests must be a whole number from 1 to 1,000,000,000." % where)
275
+ if free.get("per") not in ("hour", "day"):
276
+ errors.append('%s.free.per must be "hour" or "day".' % where)
277
+ for k in free:
278
+ if k not in ("requests", "per"):
279
+ warnings.append("%s.free.%s is not part of 0.1; clients ignore it." % (where, k))
280
+ if rule["access"] == "allow":
281
+ warnings.append('%s has a "free" allowance but its access is already "allow"; the allowance is ignored.' % where)
282
+ if rule["access"] != "paid":
283
+ if "price" in rule:
284
+ warnings.append('%s has a "price" but is not paid; the price is ignored.' % where)
285
+ return False
286
+ price = rule.get("price")
287
+ if not isinstance(price, dict):
288
+ errors.append('%s is "paid" but has no "price".' % where)
289
+ return True
290
+ amount = price.get("amount")
291
+ if not isinstance(amount, str) or re.fullmatch(r"\d+(\.\d+)?", amount) is None:
292
+ errors.append('%s.price.amount must be a decimal string, like "0.002".' % where)
293
+ elif float(amount) <= 0:
294
+ errors.append("%s.price.amount must be more than 0." % where)
295
+ if price.get("currency") != "USD":
296
+ errors.append('%s.price.currency must be "USD" in 0.1.' % where)
297
+ if price.get("per") not in ("page", "request"):
298
+ errors.append('%s.price.per must be "page" or "request".' % where)
299
+ return True
300
+
301
+ def check_payment(payment, where):
302
+ if not isinstance(payment, dict):
303
+ errors.append("%s must be an object." % where)
304
+ return
305
+ if payment.get("protocol") != "x402":
306
+ errors.append('%s.protocol must be "x402" in 0.1.' % where)
307
+ if not isinstance(payment.get("endpoint"), str) or not payment["endpoint"]:
308
+ errors.append("%s.endpoint is required: where a client pays." % where)
309
+ elif not re.match(r'^https://[^\s<>"?#]+\Z', payment["endpoint"]) or not _valid_host(payment["endpoint"]):
310
+ errors.append("%s.endpoint must be an https URL with no query or fragment: the content at path P is paid for at endpoint + P." % where)
311
+
312
+ top_paid = doc.get("default") == "paid"
313
+ unpayable_path = False
314
+ uses = doc.get("uses")
315
+ if not isinstance(uses, dict):
316
+ errors.append('"uses" is required: terms for search, answer, training and agent.')
317
+ else:
318
+ for use in USES:
319
+ if use not in uses:
320
+ errors.append('"uses.%s" is missing; all four uses are required.' % use)
321
+ elif check_use(uses[use], "uses.%s" % use):
322
+ top_paid = True
323
+ for key in uses:
324
+ if key not in USES:
325
+ warnings.append('"uses.%s" is not one of the four uses; clients ignore it.' % key)
326
+
327
+ paths = doc.get("paths")
328
+ if paths is not None:
329
+ if not isinstance(paths, list):
330
+ errors.append('"paths" must be a list.')
331
+ else:
332
+ for i, entry in enumerate(paths):
333
+ where = "paths[%d]" % i
334
+ if not isinstance(entry, dict):
335
+ errors.append('%s must be an object with "match" and "uses".' % where)
336
+ continue
337
+ if not isinstance(entry.get("match"), str) or not entry["match"].startswith("/"):
338
+ errors.append('%s.match must be a path that starts with "/".' % where)
339
+ paid = False
340
+ entry_uses = entry.get("uses")
341
+ if not isinstance(entry_uses, dict) or not entry_uses:
342
+ errors.append("%s.uses must list at least one use." % where)
343
+ else:
344
+ for key, rule in entry_uses.items():
345
+ if key not in USES:
346
+ warnings.append("%s.uses.%s is not one of the four uses." % (where, key))
347
+ elif check_use(rule, "%s.uses.%s" % (where, key)):
348
+ paid = True
349
+ if "payment" in entry:
350
+ check_payment(entry["payment"], "%s.payment" % where)
351
+ elif paid:
352
+ unpayable_path = True
353
+
354
+ if "payment" in doc:
355
+ check_payment(doc["payment"], "payment")
356
+ elif top_paid or unpayable_path:
357
+ errors.append('A use is "paid" but there is no "payment": a client has no way to pay.')
358
+ if "purpose_header" in doc and doc["purpose_header"] != "AI-Purpose":
359
+ errors.append('"purpose_header" must be "AI-Purpose" in 0.1.')
360
+ if "preview" in doc:
361
+ pv = doc["preview"]
362
+ if not isinstance(pv, dict):
363
+ errors.append('"preview" must be an object.')
364
+ else:
365
+ if pv.get("delivery") not in ("must-match", "per-quote", "not-checked"):
366
+ errors.append('"preview.delivery" must be "must-match", "per-quote" or "not-checked".')
367
+ shows = pv.get("shows")
368
+ if shows is not None and (not isinstance(shows, list) or any(v not in ("title", "description", "excerpt", "words") for v in shows)):
369
+ errors.append('"preview.shows" must list "title", "description", "excerpt" and/or "words".')
370
+ if "contact" in doc and not isinstance(doc["contact"], dict):
371
+ errors.append('"contact" must be an object.')
372
+ if "identity" in doc:
373
+ ident = doc["identity"]
374
+ if not isinstance(ident, dict):
375
+ errors.append('"identity" must be an object.')
376
+ else:
377
+ v = ident.get("verify")
378
+ if v is not None and (not isinstance(v, list) or any(not isinstance(x, str) for x in v)):
379
+ errors.append('"identity.verify" must be a list of method names, like ["published-ip-ranges"].')
380
+ if "unverified" in ident and ident["unverified"] not in ("treated-as-claimed", "treated-as-unidentified"):
381
+ warnings.append('"identity.unverified" is not "treated-as-claimed" or "treated-as-unidentified".')
382
+ return errors, warnings
383
+
384
+
385
+ def _valid_host(url):
386
+ try:
387
+ return bool(urlsplit(url).hostname)
388
+ except ValueError:
389
+ return False