wbauth-sdk 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wbauth/__init__.py +45 -0
- wbauth/_http_server/__init__.py +7 -0
- wbauth/_http_server/jwks_server.py +44 -0
- wbauth/_redaction.py +19 -0
- wbauth/_smoke/__init__.py +13 -0
- wbauth/_smoke/cloudflare_debug.py +224 -0
- wbauth/adapters/__init__.py +16 -0
- wbauth/adapters/_utils.py +74 -0
- wbauth/adapters/httpx_auth.py +49 -0
- wbauth/adapters/playwright.py +80 -0
- wbauth/adapters/requests_adapter.py +93 -0
- wbauth/cli.py +644 -0
- wbauth/identity.py +299 -0
- wbauth/normalized_request.py +32 -0
- wbauth/policy/__init__.py +31 -0
- wbauth/policy/cache.py +115 -0
- wbauth/policy/errors.py +34 -0
- wbauth/policy/inspector.py +296 -0
- wbauth/policy/parsers/__init__.py +12 -0
- wbauth/policy/parsers/ai_txt.py +65 -0
- wbauth/policy/parsers/llms_txt.py +70 -0
- wbauth/policy/parsers/robots.py +63 -0
- wbauth/policy/parsers/signing_directory.py +45 -0
- wbauth/policy/policy.py +122 -0
- wbauth/policy/verdict.py +168 -0
- wbauth/py.typed +0 -0
- wbauth/signer.py +191 -0
- wbauth_sdk-0.2.0.dist-info/METADATA +54 -0
- wbauth_sdk-0.2.0.dist-info/RECORD +31 -0
- wbauth_sdk-0.2.0.dist-info/WHEEL +4 -0
- wbauth_sdk-0.2.0.dist-info/entry_points.txt +2 -0
wbauth/__init__.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""wbauth: Web Bot Auth (RFC 9421) Python SDK.
|
|
2
|
+
|
|
3
|
+
Phase 1 (current): identity + signer + JWKS + CLI keygen.
|
|
4
|
+
Phase 2: HTTP-client adapters + policy inspector.
|
|
5
|
+
"""
|
|
6
|
+
from .adapters import (
|
|
7
|
+
WebBotAuth,
|
|
8
|
+
WebBotAuthAdapter,
|
|
9
|
+
WebBotAuthTransport,
|
|
10
|
+
attach_signing,
|
|
11
|
+
signed_session,
|
|
12
|
+
)
|
|
13
|
+
from .identity import Identity, KeyPair
|
|
14
|
+
from .normalized_request import NormalizedRequest
|
|
15
|
+
from .policy import (
|
|
16
|
+
AiTxtResult,
|
|
17
|
+
LlmsTxtResult,
|
|
18
|
+
RobotsResult,
|
|
19
|
+
SigningDirectoryResult,
|
|
20
|
+
SitePolicy,
|
|
21
|
+
inspect,
|
|
22
|
+
)
|
|
23
|
+
from .signer import SignatureHeaders, sign
|
|
24
|
+
|
|
25
|
+
__version__ = "0.2.0"
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"AiTxtResult",
|
|
29
|
+
"Identity",
|
|
30
|
+
"KeyPair",
|
|
31
|
+
"LlmsTxtResult",
|
|
32
|
+
"NormalizedRequest",
|
|
33
|
+
"RobotsResult",
|
|
34
|
+
"SignatureHeaders",
|
|
35
|
+
"SigningDirectoryResult",
|
|
36
|
+
"SitePolicy",
|
|
37
|
+
"WebBotAuth",
|
|
38
|
+
"WebBotAuthAdapter",
|
|
39
|
+
"WebBotAuthTransport",
|
|
40
|
+
"__version__",
|
|
41
|
+
"attach_signing",
|
|
42
|
+
"inspect",
|
|
43
|
+
"sign",
|
|
44
|
+
"signed_session",
|
|
45
|
+
]
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Internal HTTP server submodule for `wbauth serve` (CLI-05, D-50).
|
|
2
|
+
|
|
3
|
+
Stdlib http.server only — NO Flask, NO FastAPI, NO Starlette.
|
|
4
|
+
The 30-LOC executable budget is enforced by the plan's acceptance gate.
|
|
5
|
+
This module is internal (leading underscore) so it doesn't show up in
|
|
6
|
+
`from wbauth import *` and isn't part of the public API surface.
|
|
7
|
+
"""
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Tiny static JWKS server for self-hosters (D-50 — ≤30 LOC executable)."""
|
|
2
|
+
# Serves a single JWKS file at /.well-known/http-message-signatures-directory/{kid}
|
|
3
|
+
# with content-type per draft-meunier-http-message-signatures-directory-05.
|
|
4
|
+
# NO registration, NO list endpoints — the hosted Worker
|
|
5
|
+
# (https://wbauth.silov801.workers.dev) is the full directory.
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
import http.server
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
CONTENT_TYPE = "application/http-message-signatures-directory+json"
|
|
13
|
+
PATH_RE = re.compile(r"^/\.well-known/http-message-signatures-directory/([^/]+)$")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def make_handler(jwks_path: Path):
|
|
17
|
+
jwks_bytes = jwks_path.read_bytes()
|
|
18
|
+
jwks = json.loads(jwks_bytes)
|
|
19
|
+
served_kids = {k["kid"] for k in jwks.get("keys", [])}
|
|
20
|
+
|
|
21
|
+
class JWKSHandler(http.server.BaseHTTPRequestHandler):
|
|
22
|
+
def do_GET(self):
|
|
23
|
+
m = PATH_RE.match(self.path)
|
|
24
|
+
if not m or m.group(1) not in served_kids:
|
|
25
|
+
self.send_error(404, "kid not served by this JWKS")
|
|
26
|
+
return
|
|
27
|
+
self.send_response(200)
|
|
28
|
+
self.send_header("content-type", CONTENT_TYPE)
|
|
29
|
+
self.send_header("content-length", str(len(jwks_bytes)))
|
|
30
|
+
self.send_header("cache-control", "public, max-age=300")
|
|
31
|
+
self.end_headers()
|
|
32
|
+
self.wfile.write(jwks_bytes)
|
|
33
|
+
|
|
34
|
+
def log_message(self, format, *args):
|
|
35
|
+
pass # quiet stdlib request log
|
|
36
|
+
|
|
37
|
+
return JWKSHandler
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def serve(jwks_path: str | Path, port: int = 8080) -> None:
|
|
41
|
+
handler = make_handler(Path(jwks_path))
|
|
42
|
+
server = http.server.ThreadingHTTPServer(("0.0.0.0", port), handler)
|
|
43
|
+
print(f"Serving JWKS from {jwks_path} on port {port}")
|
|
44
|
+
server.serve_forever()
|
wbauth/_redaction.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Helpers for REDACTED repr/str on objects holding key material.
|
|
2
|
+
|
|
3
|
+
Greppable for security review: anywhere __repr__/__str__ touch keys, reuse this.
|
|
4
|
+
The literal string "REDACTED" appears in the produced repr so log scrapers can
|
|
5
|
+
flag it (and so test_repr_returns_REDACTED can assert on it).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def redacted_repr(class_name: str, **public_fields: object) -> str:
|
|
11
|
+
"""Build a `<ClassName REDACTED key1='v1' key2='v2'>` style repr.
|
|
12
|
+
|
|
13
|
+
Pass only NON-secret fields as kwargs. The output:
|
|
14
|
+
- Always begins with `<{class_name} REDACTED ` so the literal token is
|
|
15
|
+
detectable by log filters and unit tests.
|
|
16
|
+
- Renders each public field via `!r` so quotes/escapes are explicit.
|
|
17
|
+
"""
|
|
18
|
+
parts = " ".join(f"{k}={v!r}" for k, v in public_fields.items())
|
|
19
|
+
return f"<{class_name} REDACTED {parts}>"
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Internal smoke tests for live conformance against external services.
|
|
2
|
+
|
|
3
|
+
Not part of the public API. Submodules are intended to be runnable as
|
|
4
|
+
``python -m wbauth._smoke.<name>`` from CI workflows. They exit non-zero on
|
|
5
|
+
any failure (so a CI step fails loudly).
|
|
6
|
+
|
|
7
|
+
Currently:
|
|
8
|
+
|
|
9
|
+
- ``cloudflare_debug``: signs a request via :class:`wbauth.Identity` (using
|
|
10
|
+
the publicly-known RFC 9421 Appendix B.1.4 test key) and POSTs it to
|
|
11
|
+
Cloudflare's ``crawltest.com/cdn-cgi/web-bot-auth`` debug verifier.
|
|
12
|
+
Asserts HTTP 200. This is the Phase 1 hard exit criterion (IDENT-05).
|
|
13
|
+
"""
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""Sign a request with the publicly-known RFC 9421 test key, hit Cloudflare's
|
|
2
|
+
OPEN-spec verifier at ``http-message-signatures-example.research.cloudflare.com/``,
|
|
3
|
+
assert HTTP 200 + success banner.
|
|
4
|
+
|
|
5
|
+
Designed to be run as ``python -m wbauth._smoke.cloudflare_debug``. Exits
|
|
6
|
+
non-zero on any failure (CI gate per IDENT-05). Prints diagnostic information
|
|
7
|
+
to stderr on failure so the CI log is self-explanatory.
|
|
8
|
+
|
|
9
|
+
Plan 02-03 refactor (preserves Phase-1 behavior):
|
|
10
|
+
|
|
11
|
+
The original ``run() -> int`` did sign + GET + parse-banner inline. Plan 02-03
|
|
12
|
+
introduces ``wbauth verify --domain <domain>`` (CLI-03), which needs the
|
|
13
|
+
same machinery returning a structured dict instead of printing diagnostics.
|
|
14
|
+
We split the shared probe into ``_probe_verifier(identity) -> dict`` and add
|
|
15
|
+
``run_against_domain(domain, identity_path) -> dict`` for the CLI; ``run()``
|
|
16
|
+
is preserved 1:1 for the daily cron at ``.github/workflows/cloudflare-debug.yml``
|
|
17
|
+
(it now calls ``_probe_verifier`` internally and produces identical output).
|
|
18
|
+
|
|
19
|
+
Why NOT crawltest.com (the original plan choice):
|
|
20
|
+
====================================================
|
|
21
|
+
|
|
22
|
+
RESEARCH §6 and CONTEXT.md both pointed at ``https://crawltest.com/cdn-cgi/web-bot-auth``
|
|
23
|
+
as the exit gate. Empirical investigation during Plan 04 implementation
|
|
24
|
+
(2026-05-03) revealed that endpoint is the **closed verified-bots** verifier
|
|
25
|
+
which requires the kid to be registered via Cloudflare's Bot Submission Form
|
|
26
|
+
(manual dashboard step + Cloudflare-side approval — out of Phase 1 scope).
|
|
27
|
+
|
|
28
|
+
$ curl https://crawltest.com/cdn-cgi/web-bot-auth -H 'Signature: <valid-sig>' \\
|
|
29
|
+
-H 'Signature-Input: <...>' \\
|
|
30
|
+
-H 'Signature-Agent: "<jwks>"'
|
|
31
|
+
HTTP/2 401
|
|
32
|
+
unknown public key or unknown verified bot ID for keyid
|
|
33
|
+
|
|
34
|
+
This 401 happens even though the cryptographic signature is valid AND the kid
|
|
35
|
+
is published in the Signature-Agent JWKS — Cloudflare's verified-bots gate
|
|
36
|
+
also requires the kid to be in their internal allowlist. Per Cloudflare's
|
|
37
|
+
docs (https://developers.cloudflare.com/bots/reference/bot-verification/web-bot-auth/
|
|
38
|
+
step 3): "Register your bot and key directory" via Bot Submission Form.
|
|
39
|
+
|
|
40
|
+
The CORRECT open-spec verifier (verified live):
|
|
41
|
+
================================================
|
|
42
|
+
|
|
43
|
+
``https://http-message-signatures-example.research.cloudflare.com/`` validates
|
|
44
|
+
incoming requests against the published RFC 9421 Appendix B.1.4 test key:
|
|
45
|
+
|
|
46
|
+
- No signature → 200 + default homepage
|
|
47
|
+
- VALID signed request → 200 + banner "You successfully authenticated as
|
|
48
|
+
owning the test public key"
|
|
49
|
+
- INVALID signed request → 200 + banner "The Signature you sent does not
|
|
50
|
+
validate against test public key"
|
|
51
|
+
|
|
52
|
+
So the verifier ALWAYS returns 200; success/failure is in the response BODY.
|
|
53
|
+
This module checks for the success banner and exits non-zero if absent. This
|
|
54
|
+
is the actual open-spec exit gate that proves end-to-end RFC 9421 + Web Bot
|
|
55
|
+
Auth conformance without depending on Cloudflare's closed verified-bots gate.
|
|
56
|
+
|
|
57
|
+
Phase 1 hard exit criterion (IDENT-05) restated:
|
|
58
|
+
================================================
|
|
59
|
+
|
|
60
|
+
"Cloudflare's OPEN-SPEC research verifier accepts a request signed by our SDK"
|
|
61
|
+
(redirected from crawltest.com to the research server because crawltest.com
|
|
62
|
+
is the closed verified-bots gate; verified-bots registration is out of Phase 1
|
|
63
|
+
scope and tracked separately in the deferred work).
|
|
64
|
+
"""
|
|
65
|
+
from __future__ import annotations
|
|
66
|
+
|
|
67
|
+
import asyncio
|
|
68
|
+
import datetime
|
|
69
|
+
import sys
|
|
70
|
+
|
|
71
|
+
import httpx
|
|
72
|
+
|
|
73
|
+
from wbauth import Identity, NormalizedRequest, sign
|
|
74
|
+
|
|
75
|
+
CF_RESEARCH_VERIFIER_URL = (
|
|
76
|
+
"https://http-message-signatures-example.research.cloudflare.com/"
|
|
77
|
+
)
|
|
78
|
+
SUCCESS_BANNER = "You successfully authenticated as owning the test public key"
|
|
79
|
+
FAILURE_BANNER = "The Signature you sent does not validate against test public key"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
async def _probe_verifier(identity: Identity) -> dict:
|
|
83
|
+
"""Sign a probe request with ``identity`` and check Cloudflare's research verifier.
|
|
84
|
+
|
|
85
|
+
Returns a structured result dict consumed by both ``run()`` (for the daily
|
|
86
|
+
cron diagnostic output) and ``run_against_domain()`` (for the CLI).
|
|
87
|
+
|
|
88
|
+
The dict carries ``signature_input`` / ``signature`` / ``signature_agent``
|
|
89
|
+
so ``run()`` can print them on failure for diagnostics; CLI callers
|
|
90
|
+
(``wbauth verify --json``) MUST strip those keys before emitting JSON
|
|
91
|
+
(see T-02-03-02 in the plan's threat model — handled in ``cli.py``).
|
|
92
|
+
|
|
93
|
+
Result shape:
|
|
94
|
+
{
|
|
95
|
+
"result": "pass" | "fail",
|
|
96
|
+
"exit_code": 0 | 2,
|
|
97
|
+
"kid": "<identity.kid>",
|
|
98
|
+
"status": <int>,
|
|
99
|
+
"banner": "<SUCCESS|FAILURE|(no banner)>",
|
|
100
|
+
"verifier_url": CF_RESEARCH_VERIFIER_URL,
|
|
101
|
+
"signature_input": <str — internal-only, callers must strip>,
|
|
102
|
+
"signature": <str — internal-only, callers must strip>,
|
|
103
|
+
"signature_agent": <str — internal-only, callers must strip>,
|
|
104
|
+
}
|
|
105
|
+
"""
|
|
106
|
+
req = NormalizedRequest(method="GET", url=CF_RESEARCH_VERIFIER_URL, headers={})
|
|
107
|
+
sig = sign(req, identity, created=datetime.datetime.now(datetime.timezone.utc))
|
|
108
|
+
|
|
109
|
+
async with httpx.AsyncClient(timeout=10.0) as client:
|
|
110
|
+
response = await client.get(
|
|
111
|
+
CF_RESEARCH_VERIFIER_URL,
|
|
112
|
+
headers={
|
|
113
|
+
"Signature": sig.signature,
|
|
114
|
+
"Signature-Input": sig.signature_input,
|
|
115
|
+
"Signature-Agent": sig.signature_agent,
|
|
116
|
+
},
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# Pass only when the verifier explicitly says success AND status is 200.
|
|
120
|
+
# Anything else — failure banner, neither banner, or non-200 — is a fail.
|
|
121
|
+
if response.status_code == 200 and SUCCESS_BANNER in response.text:
|
|
122
|
+
result_kind = "pass"
|
|
123
|
+
banner = SUCCESS_BANNER
|
|
124
|
+
elif FAILURE_BANNER in response.text:
|
|
125
|
+
result_kind = "fail"
|
|
126
|
+
banner = FAILURE_BANNER
|
|
127
|
+
else:
|
|
128
|
+
result_kind = "fail"
|
|
129
|
+
banner = "(no banner)"
|
|
130
|
+
|
|
131
|
+
return {
|
|
132
|
+
"result": result_kind,
|
|
133
|
+
"exit_code": {"pass": 0, "fail": 2}[result_kind],
|
|
134
|
+
"kid": identity.kid,
|
|
135
|
+
"status": response.status_code,
|
|
136
|
+
"banner": banner,
|
|
137
|
+
"verifier_url": CF_RESEARCH_VERIFIER_URL,
|
|
138
|
+
"signature_input": sig.signature_input,
|
|
139
|
+
"signature": sig.signature,
|
|
140
|
+
"signature_agent": sig.signature_agent,
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
async def run_against_domain(
|
|
145
|
+
domain: str, identity_path: str | None = None
|
|
146
|
+
) -> dict:
|
|
147
|
+
"""Async helper for ``wbauth verify --domain <domain>`` (CLI-03).
|
|
148
|
+
|
|
149
|
+
v1: ALWAYS uses ``Identity.from_test_key`` regardless of ``identity_path``
|
|
150
|
+
(open question #5 resolution). The Cloudflare research verifier only
|
|
151
|
+
validates the RFC 9421 Appendix B.1.4 test key; using a real user key
|
|
152
|
+
produces the FAILURE banner. ``identity_path`` is preserved in the
|
|
153
|
+
signature for forward compatibility — Phase 3+ wires it to a
|
|
154
|
+
domain-specific verifier path after directory registration.
|
|
155
|
+
|
|
156
|
+
The ``domain`` arg is informational in v1: surfaced in the result dict
|
|
157
|
+
(so ``--json`` consumers see it) but does NOT change the HTTP target
|
|
158
|
+
(always CF_RESEARCH_VERIFIER_URL). The CLI prints a clarifying note.
|
|
159
|
+
"""
|
|
160
|
+
identity = Identity.from_test_key(signature_agent_url=CF_RESEARCH_VERIFIER_URL)
|
|
161
|
+
result = await _probe_verifier(identity)
|
|
162
|
+
result["domain"] = domain
|
|
163
|
+
return result
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def run() -> int:
|
|
167
|
+
"""Phase-1 daily-cron entry. Preserved 1:1 modulo the internal refactor.
|
|
168
|
+
|
|
169
|
+
Calls ``_probe_verifier`` and reproduces the original Phase-1 stdout/stderr
|
|
170
|
+
diagnostic shape so ``.github/workflows/cloudflare-debug.yml`` keeps
|
|
171
|
+
working unchanged. Returns 0 on pass, 1 on fail (matches the Phase-1
|
|
172
|
+
contract — NOT the Plan 02-03 CLI exit-code matrix; this entry is for the
|
|
173
|
+
cron, not the user-facing CLI).
|
|
174
|
+
"""
|
|
175
|
+
identity = Identity.from_test_key(signature_agent_url=CF_RESEARCH_VERIFIER_URL)
|
|
176
|
+
result = asyncio.run(_probe_verifier(identity))
|
|
177
|
+
|
|
178
|
+
# Pass: same OK line as Phase 1 — keeps the cron log shape stable.
|
|
179
|
+
if result["result"] == "pass":
|
|
180
|
+
print(
|
|
181
|
+
f"OK: Cloudflare research verifier accepted (status={result['status']}, "
|
|
182
|
+
f"kid={result['kid']}, banner='{SUCCESS_BANNER}')"
|
|
183
|
+
)
|
|
184
|
+
return 0
|
|
185
|
+
|
|
186
|
+
# Fail diagnostics — three sub-cases preserved from the Phase-1 module so
|
|
187
|
+
# the CI log keeps the same self-explanatory shape on regression.
|
|
188
|
+
if result["status"] != 200:
|
|
189
|
+
print(
|
|
190
|
+
"FAIL: Cloudflare research verifier returned non-200 (network or CDN issue?).\n"
|
|
191
|
+
f" status={result['status']}\n"
|
|
192
|
+
f" kid={result['kid']}\n"
|
|
193
|
+
f" Signature-Input={result['signature_input']}\n"
|
|
194
|
+
f" Signature={result['signature']}\n"
|
|
195
|
+
f" Signature-Agent={result['signature_agent']}",
|
|
196
|
+
file=sys.stderr,
|
|
197
|
+
)
|
|
198
|
+
elif result["banner"] == FAILURE_BANNER:
|
|
199
|
+
print(
|
|
200
|
+
"FAIL: Cloudflare research verifier rejected our signature.\n"
|
|
201
|
+
" banner: 'The Signature you sent does not validate against test public key'\n"
|
|
202
|
+
f" kid={result['kid']}\n"
|
|
203
|
+
f" Signature-Input={result['signature_input']}\n"
|
|
204
|
+
f" Signature={result['signature']}\n"
|
|
205
|
+
f" Signature-Agent={result['signature_agent']}\n"
|
|
206
|
+
" Diagnosis: signer regression. Check RESEARCH Pitfalls 1, 2, 6.\n"
|
|
207
|
+
" Re-run pytest test_signer to confirm regression detection.",
|
|
208
|
+
file=sys.stderr,
|
|
209
|
+
)
|
|
210
|
+
else:
|
|
211
|
+
# Neither banner — verifier may have changed shape.
|
|
212
|
+
print(
|
|
213
|
+
"FAIL: Cloudflare research verifier response unexpected (no success or "
|
|
214
|
+
"failure banner). Verifier may have changed; investigate.\n"
|
|
215
|
+
f" status={result['status']}\n"
|
|
216
|
+
f" kid={result['kid']}",
|
|
217
|
+
file=sys.stderr,
|
|
218
|
+
)
|
|
219
|
+
# Phase-1 contract: 1 on fail (not 2 — the CLI matrix is separate).
|
|
220
|
+
return 1
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
if __name__ == "__main__":
|
|
224
|
+
sys.exit(run())
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Public adapter surface for wbauth (Phase 2).
|
|
2
|
+
|
|
3
|
+
Re-exports the three drop-in adapters so users can write:
|
|
4
|
+
from wbauth.adapters import WebBotAuth, WebBotAuthAdapter, attach_signing
|
|
5
|
+
"""
|
|
6
|
+
from .httpx_auth import WebBotAuth
|
|
7
|
+
from .playwright import attach_signing
|
|
8
|
+
from .requests_adapter import WebBotAuthAdapter, WebBotAuthTransport, signed_session
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"WebBotAuth",
|
|
12
|
+
"WebBotAuthAdapter",
|
|
13
|
+
"WebBotAuthTransport",
|
|
14
|
+
"attach_signing",
|
|
15
|
+
"signed_session",
|
|
16
|
+
]
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Shared adapter helpers (internal — not part of the public surface).
|
|
2
|
+
|
|
3
|
+
The signer auto-includes ``content-digest`` in covered components for
|
|
4
|
+
POST/PUT/PATCH with a body; adapters call :func:`ensure_content_digest`
|
|
5
|
+
before ``sign()`` so that precondition is met. RFC 9530 sha-256 in
|
|
6
|
+
structured-fields form (``sha-256=:<b64>:``).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import base64
|
|
11
|
+
import hashlib
|
|
12
|
+
import re
|
|
13
|
+
from collections.abc import Iterable, MutableMapping
|
|
14
|
+
|
|
15
|
+
_DIGEST_METHODS = ("POST", "PUT", "PATCH")
|
|
16
|
+
|
|
17
|
+
# Default UAs set by HTTP libraries. An Identity UA replaces these, but an
|
|
18
|
+
# explicit caller-supplied UA always wins.
|
|
19
|
+
_LIBRARY_DEFAULT_UA = re.compile(r"^(python-requests|python-httpx|python-urllib)/", re.I)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def ensure_content_digest(method: str, headers: dict, body: bytes | None) -> None:
|
|
23
|
+
"""Add ``Content-Digest`` for POST/PUT/PATCH with a non-empty body.
|
|
24
|
+
|
|
25
|
+
No-op when the body is empty, the method carries no body, or the caller
|
|
26
|
+
already supplied a Content-Digest (case-insensitive).
|
|
27
|
+
"""
|
|
28
|
+
if not body:
|
|
29
|
+
return
|
|
30
|
+
if method.upper() not in _DIGEST_METHODS:
|
|
31
|
+
return
|
|
32
|
+
if any(k.lower() == "content-digest" for k in headers):
|
|
33
|
+
return
|
|
34
|
+
if not isinstance(body, (bytes, bytearray, memoryview)):
|
|
35
|
+
raise TypeError(
|
|
36
|
+
f"body must be bytes to compute Content-Digest, got {type(body).__name__}"
|
|
37
|
+
)
|
|
38
|
+
digest = hashlib.sha256(body).digest()
|
|
39
|
+
b64 = base64.b64encode(digest).decode("ascii")
|
|
40
|
+
headers["Content-Digest"] = f"sha-256=:{b64}:"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def should_apply_user_agent(current: str | None) -> bool:
|
|
44
|
+
"""True when the current UA is absent or a library default."""
|
|
45
|
+
return not current or bool(_LIBRARY_DEFAULT_UA.match(current))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def apply_user_agent(headers: MutableMapping[str, str], user_agent: str | None) -> None:
|
|
49
|
+
"""Set ``User-Agent`` unless the caller explicitly chose a non-default one."""
|
|
50
|
+
if not user_agent:
|
|
51
|
+
return
|
|
52
|
+
key = next((k for k in headers if k.lower() == "user-agent"), None)
|
|
53
|
+
current = headers[key] if key is not None else None
|
|
54
|
+
if should_apply_user_agent(current):
|
|
55
|
+
if key is not None:
|
|
56
|
+
del headers[key]
|
|
57
|
+
headers["User-Agent"] = user_agent
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def host_matches(hostname: str, allow: Iterable[str]) -> bool:
|
|
61
|
+
"""Match ``hostname`` against exact hosts or ``*.example.com`` wildcards.
|
|
62
|
+
|
|
63
|
+
A wildcard also matches the bare apex (``*.example.com`` ⊇ ``example.com``).
|
|
64
|
+
"""
|
|
65
|
+
h = hostname.lower().rstrip(".")
|
|
66
|
+
for raw in allow:
|
|
67
|
+
a = raw.lower().rstrip(".")
|
|
68
|
+
if a.startswith("*."):
|
|
69
|
+
base = a[2:]
|
|
70
|
+
if h == base or h.endswith("." + base):
|
|
71
|
+
return True
|
|
72
|
+
elif h == a:
|
|
73
|
+
return True
|
|
74
|
+
return False
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""httpx ``Auth`` adapter: signs every request (sync and async clients).
|
|
2
|
+
|
|
3
|
+
client = httpx.Client(auth=WebBotAuth(identity))
|
|
4
|
+
|
|
5
|
+
Each redirect hop is re-signed for its own ``@authority``; stale signature
|
|
6
|
+
headers from the previous hop are replaced, never merged.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
from wbauth.adapters._utils import ensure_content_digest, should_apply_user_agent
|
|
13
|
+
from wbauth.normalized_request import NormalizedRequest
|
|
14
|
+
from wbauth.signer import sign
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class WebBotAuth(httpx.Auth):
|
|
18
|
+
"""Drop-in ``httpx.Auth`` that adds Web Bot Auth signatures."""
|
|
19
|
+
|
|
20
|
+
requires_request_body = True # body must be readable for content-digest
|
|
21
|
+
|
|
22
|
+
def __init__(self, identity):
|
|
23
|
+
self._identity = identity
|
|
24
|
+
|
|
25
|
+
def sync_auth_flow(self, request):
|
|
26
|
+
self._sign(request)
|
|
27
|
+
yield request
|
|
28
|
+
|
|
29
|
+
async def async_auth_flow(self, request):
|
|
30
|
+
self._sign(request)
|
|
31
|
+
yield request # signing is pure CPU
|
|
32
|
+
|
|
33
|
+
def _sign(self, request: httpx.Request) -> None:
|
|
34
|
+
body = request.content if request.content else None
|
|
35
|
+
headers = dict(request.headers)
|
|
36
|
+
ensure_content_digest(request.method, headers, body)
|
|
37
|
+
if "Content-Digest" in headers:
|
|
38
|
+
request.headers["Content-Digest"] = headers["Content-Digest"]
|
|
39
|
+
normalized = NormalizedRequest(
|
|
40
|
+
method=request.method, url=str(request.url),
|
|
41
|
+
headers=headers, body=body,
|
|
42
|
+
)
|
|
43
|
+
sig = sign(normalized, self._identity)
|
|
44
|
+
request.headers["Signature"] = sig.signature
|
|
45
|
+
request.headers["Signature-Input"] = sig.signature_input
|
|
46
|
+
request.headers["Signature-Agent"] = sig.signature_agent
|
|
47
|
+
ua = getattr(self._identity, "user_agent", None)
|
|
48
|
+
if ua and should_apply_user_agent(request.headers.get("user-agent")):
|
|
49
|
+
request.headers["User-Agent"] = ua
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Playwright async helper for Web Bot Auth (ADAPT-03).
|
|
2
|
+
|
|
3
|
+
Registers a page.route handler that intercepts outgoing requests, signs the
|
|
4
|
+
matching ones via wbauth.sign(), and continues with the signed headers.
|
|
5
|
+
Stateless per D-15; async-only per D-13.
|
|
6
|
+
|
|
7
|
+
Privacy: without ``hosts`` every http(s) request is signed — including
|
|
8
|
+
third-party analytics / CDN hosts, which then learn your agent identity.
|
|
9
|
+
Restrict signing to the sites you target::
|
|
10
|
+
|
|
11
|
+
await attach_signing(page, identity, hosts=["example.com", "*.example.org"])
|
|
12
|
+
|
|
13
|
+
Non-http(s) requests (data:, blob:, chrome-extension:) are never signed.
|
|
14
|
+
Playwright disables the HTTP cache while a route is active.
|
|
15
|
+
|
|
16
|
+
PITFALL 6: Call attach_signing(page, identity) BEFORE the first
|
|
17
|
+
page.goto()/page.click() that should produce signed requests.
|
|
18
|
+
|
|
19
|
+
PITFALL 7 (iframe coverage): page.route("**/*", handler) covers all
|
|
20
|
+
sub-frame requests by default. If the target site uses Service Workers,
|
|
21
|
+
set serviceWorkers="block" on the BrowserContext (Playwright option).
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from collections.abc import Callable, Iterable
|
|
26
|
+
from urllib.parse import urlsplit
|
|
27
|
+
|
|
28
|
+
from wbauth.adapters._utils import apply_user_agent, ensure_content_digest, host_matches
|
|
29
|
+
from wbauth.normalized_request import NormalizedRequest
|
|
30
|
+
from wbauth.signer import sign
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
async def attach_signing(
|
|
34
|
+
page,
|
|
35
|
+
identity,
|
|
36
|
+
*,
|
|
37
|
+
hosts: Iterable[str] | None = None,
|
|
38
|
+
should_sign: Callable[[str], bool] | None = None,
|
|
39
|
+
pattern: str = "**/*",
|
|
40
|
+
) -> None:
|
|
41
|
+
"""Register a route handler on ``page`` that signs matching requests.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
hosts: Only sign requests to these hosts (``example.com``,
|
|
45
|
+
``*.example.com``). ``None`` = sign every http(s) request.
|
|
46
|
+
should_sign: Custom predicate on the request URL; overrides ``hosts``.
|
|
47
|
+
pattern: Route glob passed to ``page.route``.
|
|
48
|
+
"""
|
|
49
|
+
allow = list(hosts) if hosts is not None else None
|
|
50
|
+
|
|
51
|
+
def _decide(url: str) -> bool:
|
|
52
|
+
parts = urlsplit(url)
|
|
53
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
54
|
+
return False
|
|
55
|
+
if should_sign is not None:
|
|
56
|
+
return bool(should_sign(url))
|
|
57
|
+
if allow is not None:
|
|
58
|
+
return host_matches(parts.hostname, allow)
|
|
59
|
+
return True
|
|
60
|
+
|
|
61
|
+
async def _handler(route, request):
|
|
62
|
+
if not _decide(request.url):
|
|
63
|
+
fallback = getattr(route, "fallback", None)
|
|
64
|
+
await (fallback() if fallback is not None else route.continue_())
|
|
65
|
+
return
|
|
66
|
+
headers = dict(await request.all_headers())
|
|
67
|
+
body = request.post_data_buffer
|
|
68
|
+
ensure_content_digest(request.method, headers, body)
|
|
69
|
+
normalized = NormalizedRequest(
|
|
70
|
+
method=request.method, url=request.url,
|
|
71
|
+
headers=headers, body=body,
|
|
72
|
+
)
|
|
73
|
+
sig = sign(normalized, identity)
|
|
74
|
+
headers["Signature"] = sig.signature
|
|
75
|
+
headers["Signature-Input"] = sig.signature_input
|
|
76
|
+
headers["Signature-Agent"] = sig.signature_agent
|
|
77
|
+
apply_user_agent(headers, getattr(identity, "user_agent", None))
|
|
78
|
+
await route.continue_(headers=headers)
|
|
79
|
+
|
|
80
|
+
await page.route(pattern, _handler)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""requests ``AuthBase`` adapter: signs every prepared request.
|
|
2
|
+
|
|
3
|
+
requests.get(url, auth=WebBotAuthAdapter(identity))
|
|
4
|
+
session.auth = WebBotAuthAdapter(identity)
|
|
5
|
+
|
|
6
|
+
Streaming bodies (generators, file objects) are buffered once so the
|
|
7
|
+
Content-Digest can be computed; the buffered bytes are what gets sent.
|
|
8
|
+
|
|
9
|
+
**Redirects.** ``requests`` does not re-run per-request ``auth=`` on
|
|
10
|
+
redirect hops, so the first hop's signature (bound to the first host's
|
|
11
|
+
``@authority``) would be replayed to the redirect target. Use
|
|
12
|
+
:func:`signed_session` — or mount :class:`WebBotAuthTransport` yourself —
|
|
13
|
+
to re-sign every hop::
|
|
14
|
+
|
|
15
|
+
session = signed_session(identity)
|
|
16
|
+
session.get("https://example.com/")
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import requests
|
|
21
|
+
from requests.adapters import HTTPAdapter
|
|
22
|
+
from requests.auth import AuthBase
|
|
23
|
+
|
|
24
|
+
from wbauth.adapters._utils import ensure_content_digest, should_apply_user_agent
|
|
25
|
+
from wbauth.normalized_request import NormalizedRequest
|
|
26
|
+
from wbauth.signer import sign
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _buffer_body(prepared_request) -> bytes | None:
|
|
30
|
+
body = prepared_request.body
|
|
31
|
+
if body is None or isinstance(body, bytes):
|
|
32
|
+
return body
|
|
33
|
+
if isinstance(body, str):
|
|
34
|
+
return body.encode("utf-8")
|
|
35
|
+
if isinstance(body, (bytearray, memoryview)):
|
|
36
|
+
return bytes(body)
|
|
37
|
+
if hasattr(body, "read"):
|
|
38
|
+
data = body.read()
|
|
39
|
+
else: # iterable of chunks (generator, list, …)
|
|
40
|
+
data = b"".join(c.encode("utf-8") if isinstance(c, str) else bytes(c) for c in body)
|
|
41
|
+
if isinstance(data, str):
|
|
42
|
+
data = data.encode("utf-8")
|
|
43
|
+
prepared_request.body = data
|
|
44
|
+
prepared_request.headers.pop("Transfer-Encoding", None)
|
|
45
|
+
prepared_request.headers["Content-Length"] = str(len(data))
|
|
46
|
+
return data
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class WebBotAuthAdapter(AuthBase):
|
|
50
|
+
"""Drop-in ``requests`` auth that adds Web Bot Auth signatures."""
|
|
51
|
+
|
|
52
|
+
def __init__(self, identity):
|
|
53
|
+
self._identity = identity
|
|
54
|
+
|
|
55
|
+
def __call__(self, prepared_request):
|
|
56
|
+
body = _buffer_body(prepared_request)
|
|
57
|
+
headers = dict(prepared_request.headers)
|
|
58
|
+
ensure_content_digest(prepared_request.method, headers, body)
|
|
59
|
+
if "Content-Digest" in headers:
|
|
60
|
+
prepared_request.headers["Content-Digest"] = headers["Content-Digest"]
|
|
61
|
+
normalized = NormalizedRequest(
|
|
62
|
+
method=prepared_request.method, url=prepared_request.url,
|
|
63
|
+
headers=headers, body=body,
|
|
64
|
+
)
|
|
65
|
+
sig = sign(normalized, self._identity)
|
|
66
|
+
prepared_request.headers["Signature"] = sig.signature
|
|
67
|
+
prepared_request.headers["Signature-Input"] = sig.signature_input
|
|
68
|
+
prepared_request.headers["Signature-Agent"] = sig.signature_agent
|
|
69
|
+
ua = getattr(self._identity, "user_agent", None)
|
|
70
|
+
if ua and should_apply_user_agent(prepared_request.headers.get("User-Agent")):
|
|
71
|
+
prepared_request.headers["User-Agent"] = ua
|
|
72
|
+
return prepared_request
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class WebBotAuthTransport(HTTPAdapter):
|
|
76
|
+
"""Transport adapter that signs at send time — covers every redirect hop."""
|
|
77
|
+
|
|
78
|
+
def __init__(self, identity, *args, **kwargs):
|
|
79
|
+
self._auth = WebBotAuthAdapter(identity)
|
|
80
|
+
super().__init__(*args, **kwargs)
|
|
81
|
+
|
|
82
|
+
def send(self, request, *args, **kwargs): # type: ignore[override]
|
|
83
|
+
self._auth(request)
|
|
84
|
+
return super().send(request, *args, **kwargs)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def signed_session(identity, session: requests.Session | None = None) -> requests.Session:
|
|
88
|
+
"""Return a ``requests.Session`` whose every request (and redirect) is signed."""
|
|
89
|
+
s = session or requests.Session()
|
|
90
|
+
transport = WebBotAuthTransport(identity)
|
|
91
|
+
s.mount("https://", transport)
|
|
92
|
+
s.mount("http://", transport)
|
|
93
|
+
return s
|