wpr-http 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wpr_http-0.2.0 → wpr_http-0.2.2}/PKG-INFO +43 -2
- wpr_http-0.2.2/README.md +82 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/pyproject.toml +2 -2
- {wpr_http-0.2.0 → wpr_http-0.2.2}/tests/test_contracts.py +8 -2
- wpr_http-0.2.2/tests/test_header_policy.py +67 -0
- wpr_http-0.2.2/tests/test_native_header_policy.py +136 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http/__init__.py +1 -1
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http/client.py +6 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http/models.py +3 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http/transport.py +27 -1
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http.egg-info/PKG-INFO +43 -2
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http.egg-info/SOURCES.txt +2 -0
- wpr_http-0.2.2/wpr_http.egg-info/requires.txt +1 -0
- wpr_http-0.2.0/README.md +0 -41
- wpr_http-0.2.0/wpr_http.egg-info/requires.txt +0 -1
- {wpr_http-0.2.0 → wpr_http-0.2.2}/setup.cfg +0 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/tests/test_native_http.py +0 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http/__main__.py +0 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http/_urls.py +0 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http/exceptions.py +0 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http.egg-info/dependency_links.txt +0 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http.egg-info/entry_points.txt +0 -0
- {wpr_http-0.2.0 → wpr_http-0.2.2}/wpr_http.egg-info/top_level.txt +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wpr-http
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Standalone HTTP client using version-selected Firefox Necko/NSS Workers
|
|
5
5
|
License: MPL-2.0-compatible client; see runtime bundle notices
|
|
6
6
|
Project-URL: Repository, https://github.com/childclm/wpr-http
|
|
7
7
|
Project-URL: Documentation, https://github.com/childclm/wpr-http/blob/main/README.md
|
|
8
8
|
Requires-Python: >=3.10
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
|
-
Requires-Dist: wpr-runtime-core<0.3,>=0.2.
|
|
10
|
+
Requires-Dist: wpr-runtime-core<0.3,>=0.2.8
|
|
11
11
|
|
|
12
12
|
# WPR HTTP
|
|
13
13
|
|
|
@@ -50,3 +50,44 @@ The native network runtime is a separate download from the private
|
|
|
50
50
|
`childclm/gecko-web-runtime` Release. Installation needs access to that repository. Set
|
|
51
51
|
`WPR_GITHUB_TOKEN` or `GH_TOKEN`, or use Git Credential Manager, then run
|
|
52
52
|
`python -m wpr_http install-runtime --version 155`.
|
|
53
|
+
# Per-request Fetch Metadata policy
|
|
54
|
+
|
|
55
|
+
`header_policy="native"` is the backward-compatible default. Gecko generates
|
|
56
|
+
Fetch Metadata from the native Channel context and may overwrite explicit
|
|
57
|
+
`Sec-Fetch-*` headers. To preserve explicit metadata in the standalone HTTP
|
|
58
|
+
transport, use:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
response = await client.get(
|
|
62
|
+
"https://example.com/api",
|
|
63
|
+
headers={"Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "cross-site"},
|
|
64
|
+
header_policy="passthrough",
|
|
65
|
+
)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
The policy can also be a client default and overridden per request. It is
|
|
69
|
+
stored on prepared `Request` objects. Passthrough requires the native
|
|
70
|
+
`network_header_passthrough` capability; older Workers raise
|
|
71
|
+
`RuntimeCompatibilityError` before the request is sent.
|
|
72
|
+
|
|
73
|
+
Passthrough suppresses automatic Fetch Metadata generation for that network
|
|
74
|
+
Channel. Explicit `Sec-Fetch-*` fields survive redirects. Omitted metadata is
|
|
75
|
+
not invented. This is semantic header preservation, not byte-for-byte HTTP
|
|
76
|
+
replay: Host, Content-Length, HTTP/2 normalization and Gecko's credential,
|
|
77
|
+
Cookie and redirect rules remain in effect. In particular this option does
|
|
78
|
+
not restore Authorization or arbitrary sensitive headers on redirects. It
|
|
79
|
+
does not turn the request into page `fetch()` or enable CORS preflight.
|
|
80
|
+
Browser/page Workers retain Firefox's native behavior; no global preference
|
|
81
|
+
or upstream SecFetch implementation is changed.
|
|
82
|
+
|
|
83
|
+
This requires Network Runtime 155.0.9 or later; the Python change alone does
|
|
84
|
+
not update an installed binary. Upgrade both components:
|
|
85
|
+
|
|
86
|
+
```powershell
|
|
87
|
+
uv pip install --upgrade wpr-http
|
|
88
|
+
python -m wpr_http install-runtime --version 155
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`wpr-runtime-core` 0.2.3 selects network and browser defaults independently:
|
|
92
|
+
network 155.0.9, browser 155.0.8. An explicitly pinned older runtime stays
|
|
93
|
+
pinned and raises a compatibility error for passthrough requests.
|
wpr_http-0.2.2/README.md
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# WPR HTTP
|
|
2
|
+
|
|
3
|
+
Standalone HTTP requests using real, version-selected Firefox Necko/NSS network
|
|
4
|
+
Workers. Installing this package does not install the WPR browser client.
|
|
5
|
+
The native runtime is downloaded separately and remains centrally built.
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
import asyncio
|
|
9
|
+
from wpr_http import AsyncClient
|
|
10
|
+
|
|
11
|
+
async def main():
|
|
12
|
+
async with AsyncClient(firefox_version="155") as client:
|
|
13
|
+
response = await client.get("https://www.baidu.com/")
|
|
14
|
+
async with response:
|
|
15
|
+
response.raise_for_status()
|
|
16
|
+
print(response.status_code, await response.atext())
|
|
17
|
+
|
|
18
|
+
asyncio.run(main())
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
`firefox_version` selects a genuine installed Gecko network engine, not a
|
|
22
|
+
User-Agent string or a simulated browser. Only Firefox 155 is currently built
|
|
23
|
+
and supported. Selecting another version raises an explicit compatibility error;
|
|
24
|
+
it never silently uses 155. Different installed engine versions use separate
|
|
25
|
+
runtime/profile locations and separate Worker processes.
|
|
26
|
+
|
|
27
|
+
Runtime installation and supported versions:
|
|
28
|
+
|
|
29
|
+
```powershell
|
|
30
|
+
python -m wpr_http runtimes
|
|
31
|
+
python -m wpr_http install-runtime --version 155
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
An explicit `runtime_home` can select another verified runtime installation.
|
|
35
|
+
Existing `AsyncWprHttpClient`, Request/Response/Timeout and exception names remain
|
|
36
|
+
available. `WprHttpClient` remains the historical asynchronous alias.
|
|
37
|
+
|
|
38
|
+
The native network runtime is a separate download from the private
|
|
39
|
+
`childclm/gecko-web-runtime` Release. Installation needs access to that repository. Set
|
|
40
|
+
`WPR_GITHUB_TOKEN` or `GH_TOKEN`, or use Git Credential Manager, then run
|
|
41
|
+
`python -m wpr_http install-runtime --version 155`.
|
|
42
|
+
# Per-request Fetch Metadata policy
|
|
43
|
+
|
|
44
|
+
`header_policy="native"` is the backward-compatible default. Gecko generates
|
|
45
|
+
Fetch Metadata from the native Channel context and may overwrite explicit
|
|
46
|
+
`Sec-Fetch-*` headers. To preserve explicit metadata in the standalone HTTP
|
|
47
|
+
transport, use:
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
response = await client.get(
|
|
51
|
+
"https://example.com/api",
|
|
52
|
+
headers={"Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "cross-site"},
|
|
53
|
+
header_policy="passthrough",
|
|
54
|
+
)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The policy can also be a client default and overridden per request. It is
|
|
58
|
+
stored on prepared `Request` objects. Passthrough requires the native
|
|
59
|
+
`network_header_passthrough` capability; older Workers raise
|
|
60
|
+
`RuntimeCompatibilityError` before the request is sent.
|
|
61
|
+
|
|
62
|
+
Passthrough suppresses automatic Fetch Metadata generation for that network
|
|
63
|
+
Channel. Explicit `Sec-Fetch-*` fields survive redirects. Omitted metadata is
|
|
64
|
+
not invented. This is semantic header preservation, not byte-for-byte HTTP
|
|
65
|
+
replay: Host, Content-Length, HTTP/2 normalization and Gecko's credential,
|
|
66
|
+
Cookie and redirect rules remain in effect. In particular this option does
|
|
67
|
+
not restore Authorization or arbitrary sensitive headers on redirects. It
|
|
68
|
+
does not turn the request into page `fetch()` or enable CORS preflight.
|
|
69
|
+
Browser/page Workers retain Firefox's native behavior; no global preference
|
|
70
|
+
or upstream SecFetch implementation is changed.
|
|
71
|
+
|
|
72
|
+
This requires Network Runtime 155.0.9 or later; the Python change alone does
|
|
73
|
+
not update an installed binary. Upgrade both components:
|
|
74
|
+
|
|
75
|
+
```powershell
|
|
76
|
+
uv pip install --upgrade wpr-http
|
|
77
|
+
python -m wpr_http install-runtime --version 155
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`wpr-runtime-core` 0.2.3 selects network and browser defaults independently:
|
|
81
|
+
network 155.0.9, browser 155.0.8. An explicitly pinned older runtime stays
|
|
82
|
+
pinned and raises a compatibility error for passthrough requests.
|
|
@@ -4,12 +4,12 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "wpr-http"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Standalone HTTP client using version-selected Firefox Necko/NSS Workers"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
11
11
|
license = {text = "MPL-2.0-compatible client; see runtime bundle notices"}
|
|
12
|
-
dependencies = ["wpr-runtime-core>=0.2.
|
|
12
|
+
dependencies = ["wpr-runtime-core>=0.2.8,<0.3"]
|
|
13
13
|
|
|
14
14
|
[project.urls]
|
|
15
15
|
Repository = "https://github.com/childclm/wpr-http"
|
|
@@ -119,12 +119,16 @@ def test_unpublished_version_never_silently_falls_back(monkeypatch):
|
|
|
119
119
|
|
|
120
120
|
|
|
121
121
|
def test_registration_preserves_other_engines_and_exact_builds(tmp_path, monkeypatch):
|
|
122
|
+
from wpr_runtime_core import installer
|
|
122
123
|
from wpr_runtime_core.installer import _write_config, config_path
|
|
123
124
|
|
|
124
125
|
monkeypatch.setenv("LOCALAPPDATA", str(tmp_path / "appdata"))
|
|
125
126
|
monkeypatch.delenv("WPR_RUNTIME_HOME", raising=False)
|
|
126
127
|
a, _ = make_runtime(tmp_path, "155")
|
|
127
128
|
b, _ = make_runtime(tmp_path, "156")
|
|
129
|
+
monkeypatch.setattr(installer, "NETWORK_RELEASES", {
|
|
130
|
+
"155": {"version": "155.0.6"}, "156": {"version": "156.0.6"},
|
|
131
|
+
})
|
|
128
132
|
_write_config(a, product="network")
|
|
129
133
|
_write_config(b, product="network")
|
|
130
134
|
settings = json.loads(config_path().read_text(encoding="utf-8"))
|
|
@@ -157,11 +161,13 @@ def test_catalog_selects_default_and_exact_build_without_fallback(monkeypatch):
|
|
|
157
161
|
older = deepcopy(releases["builds"]["155.0.6"])
|
|
158
162
|
older["version"] = "155.0.5"
|
|
159
163
|
releases["builds"]["155.0.5"] = older
|
|
164
|
+
missing_build = "155.999.999"
|
|
165
|
+
releases["builds"].pop(missing_build, None)
|
|
160
166
|
monkeypatch.setattr(installer, "RUNTIME_CATALOG", catalog)
|
|
161
|
-
assert installer.select_release("155")["version"] ==
|
|
167
|
+
assert installer.select_release("155", product="network")["version"] == releases.get("default_network_build", releases["default_build"])
|
|
162
168
|
assert installer.select_release("155.0.5")["version"] == "155.0.5"
|
|
163
169
|
with pytest.raises(RuntimeCompatibilityError, match="no published runtime"):
|
|
164
|
-
installer.select_release(
|
|
170
|
+
installer.select_release(missing_build)
|
|
165
171
|
|
|
166
172
|
|
|
167
173
|
@pytest.mark.parametrize("kind", ["protocol", "abi", "hash", "escape"])
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wpr_http import AsyncClient, Request, RuntimeCompatibilityError
|
|
6
|
+
from wpr_http.transport import GeckoNetworkTransport
|
|
7
|
+
from wpr_runtime_core.session import RuntimeConfig
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_request_policy_defaults_and_client_override(tmp_path):
|
|
11
|
+
config = RuntimeConfig(package_root=tmp_path, runtime_dir=tmp_path,
|
|
12
|
+
worker_exe=tmp_path / "unused-worker.exe")
|
|
13
|
+
client = AsyncClient(config=config, header_policy="passthrough")
|
|
14
|
+
assert Request("GET", "http://fixture").header_policy == "native"
|
|
15
|
+
assert client.build_request("GET", "http://fixture").header_policy == "passthrough"
|
|
16
|
+
assert client.build_request("GET", "http://fixture", header_policy="native").header_policy == "native"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@pytest.mark.parametrize("value", ["cors", "", False, None])
|
|
20
|
+
def test_invalid_policy(value):
|
|
21
|
+
with pytest.raises(ValueError, match="header_policy"):
|
|
22
|
+
Request("GET", "http://fixture", header_policy=value)
|
|
23
|
+
with pytest.raises(ValueError, match="header_policy"):
|
|
24
|
+
AsyncClient(header_policy=value)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Session:
|
|
28
|
+
def __init__(self, supported=True):
|
|
29
|
+
self.supported = supported
|
|
30
|
+
self.probes = 0
|
|
31
|
+
self.sent = []
|
|
32
|
+
|
|
33
|
+
async def request(self, payload):
|
|
34
|
+
assert payload == {"method": "runtime.capabilities"}
|
|
35
|
+
self.probes += 1
|
|
36
|
+
await asyncio.sleep(0)
|
|
37
|
+
return {"ok": True, "value": {"network_header_passthrough": self.supported}}
|
|
38
|
+
|
|
39
|
+
async def request_with_timing(self, payload):
|
|
40
|
+
self.sent.append(payload)
|
|
41
|
+
return {"ok": True}, {"test": 1}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@pytest.mark.asyncio
|
|
45
|
+
async def test_capability_is_cached_per_worker_under_concurrency():
|
|
46
|
+
transport = GeckoNetworkTransport()
|
|
47
|
+
session = Session()
|
|
48
|
+
request = Request("POST", "http://fixture", headers={"Sec-Fetch-Mode": "cors"},
|
|
49
|
+
content="body", header_policy="passthrough")
|
|
50
|
+
await asyncio.gather(*(transport._send_passthrough(session, request) for _ in range(20)))
|
|
51
|
+
assert session.probes == 1
|
|
52
|
+
assert len(session.sent) == 20
|
|
53
|
+
assert session.sent[0]["header_policy"] == "passthrough"
|
|
54
|
+
assert session.sent[0]["body"] == "body"
|
|
55
|
+
assert session.sent[0]["headers"]["Sec-Fetch-Mode"] == "cors"
|
|
56
|
+
another = Session()
|
|
57
|
+
await transport._send_passthrough(another, request)
|
|
58
|
+
assert another.probes == 1
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@pytest.mark.asyncio
|
|
62
|
+
async def test_old_worker_is_rejected_before_network_request():
|
|
63
|
+
session = Session(supported=False)
|
|
64
|
+
with pytest.raises(RuntimeCompatibilityError, match="update the native runtime"):
|
|
65
|
+
await GeckoNetworkTransport()._send_passthrough(
|
|
66
|
+
session, Request("GET", "http://fixture", header_policy="passthrough"))
|
|
67
|
+
assert session.sent == []
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Receiver-side contracts; opt in with WPR_TEST_WORKER_EXE."""
|
|
2
|
+
import asyncio
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import threading
|
|
6
|
+
from dataclasses import replace
|
|
7
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
from wpr_http import AsyncClient
|
|
13
|
+
|
|
14
|
+
pytestmark = pytest.mark.skipif(not os.environ.get("WPR_TEST_WORKER_EXE"),
|
|
15
|
+
reason="requires a freshly built native Worker")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Echo(BaseHTTPRequestHandler):
|
|
19
|
+
def log_message(self, *_):
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
def do_GET(self):
|
|
23
|
+
if self.path.startswith("/redirect"):
|
|
24
|
+
target = self.server.other if self.path == "/redirect-cross" else self.server.server_port
|
|
25
|
+
self.send_response(302)
|
|
26
|
+
self.send_header("Location", f"http://127.0.0.1:{target}/echo")
|
|
27
|
+
self.send_header("Content-Length", "0")
|
|
28
|
+
self.end_headers()
|
|
29
|
+
return
|
|
30
|
+
size = int(self.headers.get("Content-Length", "0"))
|
|
31
|
+
body = json.dumps({"headers": {k.lower(): v for k, v in self.headers.items()},
|
|
32
|
+
"body": self.rfile.read(size).decode() if size else ""}).encode()
|
|
33
|
+
self.send_response(200)
|
|
34
|
+
self.send_header("Content-Type", "application/json")
|
|
35
|
+
self.send_header("Set-Cookie", "policy=kept; Path=/")
|
|
36
|
+
self.send_header("Content-Length", str(len(body)))
|
|
37
|
+
self.end_headers()
|
|
38
|
+
self.wfile.write(body)
|
|
39
|
+
|
|
40
|
+
do_POST = do_GET
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@pytest.fixture
|
|
44
|
+
def fixture():
|
|
45
|
+
servers = [ThreadingHTTPServer(("127.0.0.1", 0), Echo) for _ in range(2)]
|
|
46
|
+
for server in servers:
|
|
47
|
+
server.other = servers[1].server_port
|
|
48
|
+
threads = [threading.Thread(target=s.serve_forever, daemon=True) for s in servers]
|
|
49
|
+
for thread in threads:
|
|
50
|
+
thread.start()
|
|
51
|
+
worker = Path(os.environ["WPR_TEST_WORKER_EXE"])
|
|
52
|
+
base = AsyncClient(firefox_version="155", runtime_home=os.environ.get("WPR_TEST_RUNTIME_HOME")).config
|
|
53
|
+
config = replace(base, worker_exe=worker, network_worker_exe=worker,
|
|
54
|
+
runtime_dir=worker.parent, proxy="")
|
|
55
|
+
try:
|
|
56
|
+
yield config, f"http://127.0.0.1:{servers[0].server_port}"
|
|
57
|
+
finally:
|
|
58
|
+
for server in servers:
|
|
59
|
+
server.shutdown()
|
|
60
|
+
server.server_close()
|
|
61
|
+
for thread in threads:
|
|
62
|
+
thread.join()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
EXPLICIT = {"Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "cross-site",
|
|
66
|
+
"Sec-Fetch-Dest": "empty", "Sec-Fetch-User": "?1", "X-Probe": "kept",
|
|
67
|
+
"Origin": "https://source.example.net", "Referer": "https://source.example.net/"}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@pytest.mark.asyncio
|
|
71
|
+
async def test_default_passthrough_and_cookie_body(fixture):
|
|
72
|
+
config, root = fixture
|
|
73
|
+
async with AsyncClient(config=config, proxy="") as client:
|
|
74
|
+
native = await (await client.get(root, headers=EXPLICIT)).ajson()
|
|
75
|
+
assert native["headers"]["sec-fetch-mode"] == "no-cors"
|
|
76
|
+
assert native["headers"]["sec-fetch-site"] == "none"
|
|
77
|
+
response = await client.post(root, headers=EXPLICIT, content="payload",
|
|
78
|
+
header_policy="passthrough")
|
|
79
|
+
actual = await response.ajson()
|
|
80
|
+
assert actual["body"] == "payload"
|
|
81
|
+
assert "policy=kept" in actual["headers"]["cookie"]
|
|
82
|
+
for name, value in EXPLICIT.items():
|
|
83
|
+
assert actual["headers"][name.lower()] == value
|
|
84
|
+
raw = await (await client.get(root, header_policy="passthrough")).ajson()
|
|
85
|
+
assert not any(k.startswith("sec-fetch-") for k in raw["headers"])
|
|
86
|
+
after = await (await client.get(root)).ajson()
|
|
87
|
+
assert after["headers"]["sec-fetch-mode"] == "no-cors"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@pytest.mark.asyncio
|
|
91
|
+
@pytest.mark.parametrize("path", ["/redirect", "/redirect-cross"])
|
|
92
|
+
async def test_redirect_preserves_metadata_not_credentials(fixture, path):
|
|
93
|
+
config, root = fixture
|
|
94
|
+
async with AsyncClient(config=config, proxy="") as client:
|
|
95
|
+
headers = {**EXPLICIT, "Authorization": "Bearer fixture-only"}
|
|
96
|
+
actual = await (await client.get(root + path, headers=headers,
|
|
97
|
+
header_policy="passthrough")).ajson()
|
|
98
|
+
for key in ("sec-fetch-mode", "sec-fetch-site", "sec-fetch-dest", "sec-fetch-user"):
|
|
99
|
+
assert actual["headers"][key] == {k.lower(): v for k, v in EXPLICIT.items()}[key]
|
|
100
|
+
if path == "/redirect-cross":
|
|
101
|
+
assert "authorization" not in actual["headers"]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@pytest.mark.asyncio
|
|
105
|
+
async def test_mixed_policies_share_one_worker_without_leakage(fixture):
|
|
106
|
+
config, root = fixture
|
|
107
|
+
async with AsyncClient(config=config, proxy="", max_workers=1, max_channels_per_worker=20) as client:
|
|
108
|
+
async def one(i):
|
|
109
|
+
policy = "passthrough" if i % 2 else "native"
|
|
110
|
+
headers = {k.lower(): v for k, v in EXPLICIT.items()} if i % 3 else EXPLICIT
|
|
111
|
+
actual = await (await client.get(root, headers=headers, header_policy=policy)).ajson()
|
|
112
|
+
expected = ("cors", "cross-site") if i % 2 else ("no-cors", "none")
|
|
113
|
+
assert (actual["headers"]["sec-fetch-mode"], actual["headers"]["sec-fetch-site"]) == expected
|
|
114
|
+
await asyncio.gather(*(one(i) for i in range(40)))
|
|
115
|
+
assert client.transport_stats["worker_count"] == 1
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@pytest.mark.asyncio
|
|
119
|
+
async def test_nonmultiplex_pipe_and_client_default(fixture):
|
|
120
|
+
config, root = fixture
|
|
121
|
+
async with AsyncClient(config=config, proxy="", header_policy="passthrough",
|
|
122
|
+
ipc_backend="threaded", release_mode="immediate", max_channels_per_worker=1) as client:
|
|
123
|
+
actual = await (await client.get(root, headers=EXPLICIT)).ajson()
|
|
124
|
+
assert actual["headers"]["sec-fetch-mode"] == "cors"
|
|
125
|
+
native = await (await client.get(root, headers=EXPLICIT, header_policy="native")).ajson()
|
|
126
|
+
assert native["headers"]["sec-fetch-mode"] == "no-cors"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@pytest.mark.asyncio
|
|
130
|
+
async def test_invalid_policy_rpc_is_rejected(fixture):
|
|
131
|
+
config, root = fixture
|
|
132
|
+
async with AsyncClient(config=config, proxy="") as client:
|
|
133
|
+
raw = await client._slots[0].session.request({"method": "network.fetch",
|
|
134
|
+
"url": root, "header_policy": "invalid-policy"})
|
|
135
|
+
assert raw["ok"] is False
|
|
136
|
+
assert raw["error"]["code"] == "INVALID_HEADER_POLICY"
|
|
@@ -83,7 +83,11 @@ class AsyncWprHttpClient:
|
|
|
83
83
|
release_mode: str = "piggyback",
|
|
84
84
|
ipc_backend: str = "iocp",
|
|
85
85
|
max_channels_per_worker: int = 20,
|
|
86
|
+
header_policy: str = "native",
|
|
86
87
|
) -> None:
|
|
88
|
+
if header_policy not in {"native", "passthrough"}:
|
|
89
|
+
raise ValueError("header_policy must be 'native' or 'passthrough'")
|
|
90
|
+
self.header_policy = header_policy
|
|
87
91
|
try:
|
|
88
92
|
worker_limit = integer_index(max_workers)
|
|
89
93
|
except TypeError:
|
|
@@ -340,6 +344,7 @@ class AsyncWprHttpClient:
|
|
|
340
344
|
params: Mapping[str, Any] | Sequence[tuple[str, Any]] | None = None,
|
|
341
345
|
timeout: float | Timeout | None | object = _UNSET,
|
|
342
346
|
follow_redirects: bool | None = None,
|
|
347
|
+
header_policy: str | None = None,
|
|
343
348
|
) -> Request:
|
|
344
349
|
"""Merge client defaults and return a prepared, inspectable request."""
|
|
345
350
|
if sum(value is not None for value in (content, data, json)) > 1:
|
|
@@ -382,6 +387,7 @@ class AsyncWprHttpClient:
|
|
|
382
387
|
headers=request_headers,
|
|
383
388
|
content=request_body,
|
|
384
389
|
timeout=selected_timeout,
|
|
390
|
+
header_policy=self.header_policy if header_policy is None else header_policy,
|
|
385
391
|
follow_redirects=(
|
|
386
392
|
self.follow_redirects
|
|
387
393
|
if follow_redirects is None
|
|
@@ -101,8 +101,11 @@ class Request:
|
|
|
101
101
|
content: str | None = None
|
|
102
102
|
timeout: Timeout | float | None = field(default_factory=Timeout)
|
|
103
103
|
follow_redirects: bool = True
|
|
104
|
+
header_policy: str = "native"
|
|
104
105
|
|
|
105
106
|
def __post_init__(self) -> None:
|
|
107
|
+
if self.header_policy not in {"native", "passthrough"}:
|
|
108
|
+
raise ValueError("header_policy must be 'native' or 'passthrough'")
|
|
106
109
|
object.__setattr__(self, "method", self.method.upper())
|
|
107
110
|
object.__setattr__(self, "headers", Headers(self.headers))
|
|
108
111
|
if not isinstance(self.timeout, Timeout):
|
|
@@ -4,8 +4,10 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import asyncio
|
|
6
6
|
import logging
|
|
7
|
+
import weakref
|
|
7
8
|
|
|
8
9
|
from wpr_runtime_core.data_plane import SharedBlob, SharedBlobDescriptor
|
|
10
|
+
from wpr_runtime_core.registry import RuntimeCompatibilityError
|
|
9
11
|
from wpr_runtime_core.session import AsyncWorkerSession as AsyncGeckoWorkerSession
|
|
10
12
|
|
|
11
13
|
from ._urls import safe_url
|
|
@@ -18,6 +20,28 @@ _logger = logging.getLogger("wpr_http")
|
|
|
18
20
|
class GeckoNetworkTransport:
|
|
19
21
|
"""One-request adapter; all Gecko protocol details stay below the client."""
|
|
20
22
|
|
|
23
|
+
def __init__(self) -> None:
|
|
24
|
+
self._header_capabilities = weakref.WeakKeyDictionary()
|
|
25
|
+
|
|
26
|
+
async def _send_passthrough(self, session, request):
|
|
27
|
+
capability_task = self._header_capabilities.get(session)
|
|
28
|
+
if capability_task is None:
|
|
29
|
+
capability_task = asyncio.create_task(session.request({"method": "runtime.capabilities"}))
|
|
30
|
+
self._header_capabilities[session] = capability_task
|
|
31
|
+
response = await asyncio.shield(capability_task)
|
|
32
|
+
if not response.get("ok") or not response.get("value", {}).get("network_header_passthrough"):
|
|
33
|
+
raise RuntimeCompatibilityError(
|
|
34
|
+
"header_policy='passthrough' requires a Network Worker with "
|
|
35
|
+
"network_header_passthrough support; update the native runtime"
|
|
36
|
+
)
|
|
37
|
+
payload = {"method": "network.fetch", "url": request.url,
|
|
38
|
+
"http_method": request.method, "headers": dict(request.headers),
|
|
39
|
+
"follow_redirects": request.follow_redirects,
|
|
40
|
+
"header_policy": "passthrough"}
|
|
41
|
+
if request.content is not None:
|
|
42
|
+
payload["body"] = request.content
|
|
43
|
+
return await session.request_with_timing(payload)
|
|
44
|
+
|
|
21
45
|
async def send(
|
|
22
46
|
self,
|
|
23
47
|
session: AsyncGeckoWorkerSession,
|
|
@@ -34,7 +58,9 @@ class GeckoNetworkTransport:
|
|
|
34
58
|
"follow_redirects": request.follow_redirects,
|
|
35
59
|
}
|
|
36
60
|
timed_fetch = getattr(session, "network_fetch_with_timing", None)
|
|
37
|
-
if
|
|
61
|
+
if request.header_policy == "passthrough":
|
|
62
|
+
operation = self._send_passthrough(session, request)
|
|
63
|
+
elif timed_fetch is None:
|
|
38
64
|
|
|
39
65
|
async def legacy_fetch():
|
|
40
66
|
return await session.network_fetch(request.url, **fetch_kwargs), {}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wpr-http
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Standalone HTTP client using version-selected Firefox Necko/NSS Workers
|
|
5
5
|
License: MPL-2.0-compatible client; see runtime bundle notices
|
|
6
6
|
Project-URL: Repository, https://github.com/childclm/wpr-http
|
|
7
7
|
Project-URL: Documentation, https://github.com/childclm/wpr-http/blob/main/README.md
|
|
8
8
|
Requires-Python: >=3.10
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
|
-
Requires-Dist: wpr-runtime-core<0.3,>=0.2.
|
|
10
|
+
Requires-Dist: wpr-runtime-core<0.3,>=0.2.8
|
|
11
11
|
|
|
12
12
|
# WPR HTTP
|
|
13
13
|
|
|
@@ -50,3 +50,44 @@ The native network runtime is a separate download from the private
|
|
|
50
50
|
`childclm/gecko-web-runtime` Release. Installation needs access to that repository. Set
|
|
51
51
|
`WPR_GITHUB_TOKEN` or `GH_TOKEN`, or use Git Credential Manager, then run
|
|
52
52
|
`python -m wpr_http install-runtime --version 155`.
|
|
53
|
+
# Per-request Fetch Metadata policy
|
|
54
|
+
|
|
55
|
+
`header_policy="native"` is the backward-compatible default. Gecko generates
|
|
56
|
+
Fetch Metadata from the native Channel context and may overwrite explicit
|
|
57
|
+
`Sec-Fetch-*` headers. To preserve explicit metadata in the standalone HTTP
|
|
58
|
+
transport, use:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
response = await client.get(
|
|
62
|
+
"https://example.com/api",
|
|
63
|
+
headers={"Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "cross-site"},
|
|
64
|
+
header_policy="passthrough",
|
|
65
|
+
)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
The policy can also be a client default and overridden per request. It is
|
|
69
|
+
stored on prepared `Request` objects. Passthrough requires the native
|
|
70
|
+
`network_header_passthrough` capability; older Workers raise
|
|
71
|
+
`RuntimeCompatibilityError` before the request is sent.
|
|
72
|
+
|
|
73
|
+
Passthrough suppresses automatic Fetch Metadata generation for that network
|
|
74
|
+
Channel. Explicit `Sec-Fetch-*` fields survive redirects. Omitted metadata is
|
|
75
|
+
not invented. This is semantic header preservation, not byte-for-byte HTTP
|
|
76
|
+
replay: Host, Content-Length, HTTP/2 normalization and Gecko's credential,
|
|
77
|
+
Cookie and redirect rules remain in effect. In particular this option does
|
|
78
|
+
not restore Authorization or arbitrary sensitive headers on redirects. It
|
|
79
|
+
does not turn the request into page `fetch()` or enable CORS preflight.
|
|
80
|
+
Browser/page Workers retain Firefox's native behavior; no global preference
|
|
81
|
+
or upstream SecFetch implementation is changed.
|
|
82
|
+
|
|
83
|
+
This requires Network Runtime 155.0.9 or later; the Python change alone does
|
|
84
|
+
not update an installed binary. Upgrade both components:
|
|
85
|
+
|
|
86
|
+
```powershell
|
|
87
|
+
uv pip install --upgrade wpr-http
|
|
88
|
+
python -m wpr_http install-runtime --version 155
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`wpr-runtime-core` 0.2.3 selects network and browser defaults independently:
|
|
92
|
+
network 155.0.9, browser 155.0.8. An explicitly pinned older runtime stays
|
|
93
|
+
pinned and raises a compatibility error for passthrough requests.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
wpr-runtime-core<0.3,>=0.2.8
|
wpr_http-0.2.0/README.md
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
# WPR HTTP
|
|
2
|
-
|
|
3
|
-
Standalone HTTP requests using real, version-selected Firefox Necko/NSS network
|
|
4
|
-
Workers. Installing this package does not install the WPR browser client.
|
|
5
|
-
The native runtime is downloaded separately and remains centrally built.
|
|
6
|
-
|
|
7
|
-
```python
|
|
8
|
-
import asyncio
|
|
9
|
-
from wpr_http import AsyncClient
|
|
10
|
-
|
|
11
|
-
async def main():
|
|
12
|
-
async with AsyncClient(firefox_version="155") as client:
|
|
13
|
-
response = await client.get("https://www.baidu.com/")
|
|
14
|
-
async with response:
|
|
15
|
-
response.raise_for_status()
|
|
16
|
-
print(response.status_code, await response.atext())
|
|
17
|
-
|
|
18
|
-
asyncio.run(main())
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
`firefox_version` selects a genuine installed Gecko network engine, not a
|
|
22
|
-
User-Agent string or a simulated browser. Only Firefox 155 is currently built
|
|
23
|
-
and supported. Selecting another version raises an explicit compatibility error;
|
|
24
|
-
it never silently uses 155. Different installed engine versions use separate
|
|
25
|
-
runtime/profile locations and separate Worker processes.
|
|
26
|
-
|
|
27
|
-
Runtime installation and supported versions:
|
|
28
|
-
|
|
29
|
-
```powershell
|
|
30
|
-
python -m wpr_http runtimes
|
|
31
|
-
python -m wpr_http install-runtime --version 155
|
|
32
|
-
```
|
|
33
|
-
|
|
34
|
-
An explicit `runtime_home` can select another verified runtime installation.
|
|
35
|
-
Existing `AsyncWprHttpClient`, Request/Response/Timeout and exception names remain
|
|
36
|
-
available. `WprHttpClient` remains the historical asynchronous alias.
|
|
37
|
-
|
|
38
|
-
The native network runtime is a separate download from the private
|
|
39
|
-
`childclm/gecko-web-runtime` Release. Installation needs access to that repository. Set
|
|
40
|
-
`WPR_GITHUB_TOKEN` or `GH_TOKEN`, or use Git Credential Manager, then run
|
|
41
|
-
`python -m wpr_http install-runtime --version 155`.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
wpr-runtime-core<0.3,>=0.2.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|