httpr 0.6.0__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {httpr-0.6.0 → httpr-0.7.1}/.github/copilot-instructions.md +1 -1
- {httpr-0.6.0 → httpr-0.7.1}/.github/workflows/benchmark.yml +1 -1
- httpr-0.7.1/.github/workflows/codspeed.yml +50 -0
- httpr-0.7.1/.github/workflows/compare.yml +153 -0
- {httpr-0.6.0 → httpr-0.7.1}/.github/workflows/mkdocs.yml +15 -12
- {httpr-0.6.0 → httpr-0.7.1}/.gitignore +4 -1
- httpr-0.7.1/CLAUDE.md +226 -0
- {httpr-0.6.0 → httpr-0.7.1}/Cargo.lock +322 -368
- {httpr-0.6.0 → httpr-0.7.1}/Cargo.toml +3 -2
- {httpr-0.6.0 → httpr-0.7.1}/PKG-INFO +66 -17
- {httpr-0.6.0 → httpr-0.7.1}/README.md +64 -16
- {httpr-0.6.0 → httpr-0.7.1}/Taskfile.yaml +13 -3
- httpr-0.7.1/benchmark/README.md +33 -0
- {httpr-0.6.0 → httpr-0.7.1}/benchmark/benchmark.py +10 -4
- httpr-0.7.1/benchmark/render_comparison.py +181 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/advanced/proxy.md +9 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/api/async-client.md +13 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/api/response.md +60 -0
- httpr-0.7.1/docs/benchmark.md +36 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/index.md +1 -1
- {httpr-0.6.0 → httpr-0.7.1}/docs/quickstart.md +17 -1
- {httpr-0.6.0 → httpr-0.7.1}/docs/tutorial/making-requests.md +8 -2
- {httpr-0.6.0 → httpr-0.7.1}/docs/tutorial/response-handling.md +12 -4
- {httpr-0.6.0 → httpr-0.7.1}/httpr/__init__.py +301 -44
- {httpr-0.6.0 → httpr-0.7.1}/httpr/httpr.pyi +51 -13
- {httpr-0.6.0 → httpr-0.7.1}/mkdocs.yml +1 -0
- {httpr-0.6.0 → httpr-0.7.1}/pyproject.toml +2 -2
- {httpr-0.6.0 → httpr-0.7.1}/src/exceptions.rs +19 -1
- httpr-0.7.1/src/lib.rs +836 -0
- httpr-0.7.1/src/lifecycle.rs +263 -0
- httpr-0.7.1/src/params.rs +128 -0
- {httpr-0.6.0 → httpr-0.7.1}/src/response.rs +76 -28
- httpr-0.7.1/src/timeout.rs +127 -0
- {httpr-0.6.0 → httpr-0.7.1}/src/traits.rs +12 -0
- {httpr-0.6.0 → httpr-0.7.1}/src/utils.rs +21 -31
- httpr-0.7.1/tests/benchmark/codspeed/README.md +43 -0
- httpr-0.7.1/tests/benchmark/codspeed/bench_server.py +272 -0
- httpr-0.7.1/tests/benchmark/codspeed/conftest.py +104 -0
- httpr-0.7.1/tests/benchmark/codspeed/test_decoding.py +114 -0
- httpr-0.7.1/tests/benchmark/codspeed/test_transport.py +225 -0
- httpr-0.7.1/tests/unit/__init__.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/test_asyncclient.py +16 -7
- httpr-0.7.1/tests/unit/test_ca_bundle.py +81 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/test_client.py +43 -0
- httpr-0.7.1/tests/unit/test_close.py +420 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/test_exceptions.py +4 -0
- httpr-0.7.1/tests/unit/test_params.py +211 -0
- httpr-0.7.1/tests/unit/test_proxy.py +137 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/test_ssl.py +3 -9
- httpr-0.7.1/tests/unit/test_streaming.py +429 -0
- httpr-0.7.1/tests/unit/test_timeout.py +261 -0
- httpr-0.7.1/uv.lock +2000 -0
- httpr-0.6.0/.github/workflows/CI.yml.backup +0 -362
- httpr-0.6.0/CLAUDE.md +0 -186
- httpr-0.6.0/benchmark/README.md +0 -17
- httpr-0.6.0/benchmark.jpg +0 -0
- httpr-0.6.0/httpr.code-workspace +0 -17
- httpr-0.6.0/src/lib.rs +0 -732
- httpr-0.6.0/tests/unit/test_streaming.py +0 -253
- httpr-0.6.0/uv.lock +0 -1927
- {httpr-0.6.0 → httpr-0.7.1}/.github/actions/set-version/action.yml +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/.github/workflows/CI.yml +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/.github/workflows/copilot-setup-steps.yml +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/.github/workflows/set_version.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/.pre-commit-config.yaml +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/LICENSE +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/benchmark/__init__.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/benchmark/benchmark_cbor.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/benchmark/server.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/advanced/cookies.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/advanced/index.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/advanced/ssl-tls.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/api/client.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/api/functions.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/api/index.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/tutorial/async.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/tutorial/authentication.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/tutorial/index.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/writings/index.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/docs/writings/posts/2025-02-24-python-http-clients-suck.md +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/httpr/py.typed +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/scripts/generate_certs.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/benchmark/__init__.py +0 -0
- {httpr-0.6.0/tests/e2e → httpr-0.7.1/tests/benchmark/codspeed}/__init__.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/benchmark/test_performance.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/conftest.py +0 -0
- {httpr-0.6.0/tests/unit → httpr-0.7.1/tests/e2e}/__init__.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/e2e/test_async.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/e2e/test_auth.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/e2e/test_redirects.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/e2e/test_ssl.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/e2e/test_streaming.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/e2e/test_uploads.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/cbor_test_server.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/httpx_conns.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/test_cbor.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/test_defs.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/test_docs.py +0 -0
- {httpr-0.6.0 → httpr-0.7.1}/tests/unit/test_response.py +0 -0
|
@@ -84,7 +84,7 @@ with client.stream("GET", url) as response:
|
|
|
84
84
|
## Critical Details
|
|
85
85
|
|
|
86
86
|
- **Body types mutually exclusive:** `content` (bytes) | `data` (form) | `json` | `files` (multipart)
|
|
87
|
-
- **SSL:** `ca_cert_file`
|
|
87
|
+
- **SSL:** `ca_cert_file` param, else `HTTPR_CA_BUNDLE` env var as default (read-only, scoped per client); `verify=False` for insecure
|
|
88
88
|
- **Proxy:** `proxy` param or `HTTPR_PROXY` env var; changing rebuilds entire client
|
|
89
89
|
- **Python types:** `IndexMap<String, String, RandomState>` in Rust ↔ `dict[str, str]` in Python
|
|
90
90
|
|
|
@@ -62,7 +62,7 @@ jobs:
|
|
|
62
62
|
- name: Run benchmarks
|
|
63
63
|
env:
|
|
64
64
|
BENCHMARK_SERVER_URL: "http://127.0.0.1:8000"
|
|
65
|
-
run: uv run pytest tests/benchmark/ --benchmark-only --benchmark-json=benchmark-results.json
|
|
65
|
+
run: uv run pytest tests/benchmark/test_performance.py --benchmark-only --benchmark-json=benchmark-results.json
|
|
66
66
|
|
|
67
67
|
- name: Store benchmark result
|
|
68
68
|
uses: benchmark-action/github-action-benchmark@v1
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: CodSpeed
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
pull_request:
|
|
8
|
+
# Lets CodSpeed trigger a backtest run to generate the initial baseline data.
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
id-token: write # OpenID Connect authentication with CodSpeed
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
benchmarks:
|
|
17
|
+
name: Run benchmarks
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v5
|
|
21
|
+
|
|
22
|
+
- uses: astral-sh/setup-uv@v7
|
|
23
|
+
with:
|
|
24
|
+
python-version: "3.12"
|
|
25
|
+
|
|
26
|
+
- name: Cache Rust build
|
|
27
|
+
uses: actions/cache@v5
|
|
28
|
+
with:
|
|
29
|
+
path: |
|
|
30
|
+
~/.cargo/registry
|
|
31
|
+
~/.cargo/git
|
|
32
|
+
target
|
|
33
|
+
key: rust-codspeed-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
|
|
34
|
+
restore-keys: |
|
|
35
|
+
rust-codspeed-${{ runner.os }}-
|
|
36
|
+
|
|
37
|
+
- name: Install dependencies
|
|
38
|
+
run: uv sync --extra dev
|
|
39
|
+
|
|
40
|
+
# Benchmarks must run against an optimised build, otherwise they measure
|
|
41
|
+
# debug assertions and unoptimised Rust rather than shipped code.
|
|
42
|
+
- name: Build Rust extension (release mode)
|
|
43
|
+
run: uv run maturin develop --release
|
|
44
|
+
|
|
45
|
+
- name: Run benchmarks
|
|
46
|
+
uses: CodSpeedHQ/action@v5
|
|
47
|
+
with:
|
|
48
|
+
mode: simulation
|
|
49
|
+
# --no-sync keeps uv from replacing the release build made above.
|
|
50
|
+
run: uv run --no-sync pytest tests/benchmark/codspeed --codspeed
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
name: Client comparison
|
|
2
|
+
|
|
3
|
+
# Weekly comparison of httpr against the latest release of every other HTTP
|
|
4
|
+
# client in benchmark/benchmark.py. Results are published to gh-pages at
|
|
5
|
+
# dev/bench/compare/ (chart, tables, CSVs) and embedded in the README and docs.
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
schedule:
|
|
9
|
+
- cron: "0 6 * * 1" # Mondays 06:00 UTC
|
|
10
|
+
workflow_dispatch:
|
|
11
|
+
inputs:
|
|
12
|
+
httpr_source:
|
|
13
|
+
description: "Which httpr to benchmark"
|
|
14
|
+
type: choice
|
|
15
|
+
default: pypi
|
|
16
|
+
options:
|
|
17
|
+
- pypi # latest release, what `pip install httpr` gives (the scheduled default)
|
|
18
|
+
- main # a wheel built from the checked-out commit
|
|
19
|
+
publish:
|
|
20
|
+
description: "Publish the results to gh-pages"
|
|
21
|
+
type: boolean
|
|
22
|
+
default: true
|
|
23
|
+
|
|
24
|
+
permissions:
|
|
25
|
+
contents: write
|
|
26
|
+
|
|
27
|
+
concurrency:
|
|
28
|
+
group: gh-pages-compare
|
|
29
|
+
cancel-in-progress: false
|
|
30
|
+
|
|
31
|
+
jobs:
|
|
32
|
+
compare:
|
|
33
|
+
name: HTTP-client comparison
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
env:
|
|
36
|
+
HTTPR_SOURCE: ${{ github.event_name == 'workflow_dispatch' && inputs.httpr_source || 'pypi' }}
|
|
37
|
+
steps:
|
|
38
|
+
- uses: actions/checkout@v5
|
|
39
|
+
|
|
40
|
+
- uses: astral-sh/setup-uv@v7
|
|
41
|
+
with:
|
|
42
|
+
python-version: "3.12"
|
|
43
|
+
|
|
44
|
+
- name: Cache Rust build
|
|
45
|
+
if: env.HTTPR_SOURCE == 'main'
|
|
46
|
+
uses: actions/cache@v5
|
|
47
|
+
with:
|
|
48
|
+
path: |
|
|
49
|
+
~/.cargo/registry
|
|
50
|
+
~/.cargo/git
|
|
51
|
+
target
|
|
52
|
+
key: rust-bench-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
|
|
53
|
+
restore-keys: |
|
|
54
|
+
rust-bench-${{ runner.os }}-
|
|
55
|
+
|
|
56
|
+
- name: Build httpr wheel (release mode)
|
|
57
|
+
if: env.HTTPR_SOURCE == 'main'
|
|
58
|
+
run: uv build --wheel --out-dir dist
|
|
59
|
+
|
|
60
|
+
- name: Start benchmark server
|
|
61
|
+
env:
|
|
62
|
+
BENCHMARK_HOST: 127.0.0.1
|
|
63
|
+
BENCHMARK_PORT: "8000"
|
|
64
|
+
run: |
|
|
65
|
+
uv run --script benchmark/server.py &
|
|
66
|
+
for i in $(seq 1 10); do
|
|
67
|
+
if curl -sf http://127.0.0.1:8000/json/1 > /dev/null 2>&1; then
|
|
68
|
+
echo "Benchmark server is ready"
|
|
69
|
+
exit 0
|
|
70
|
+
fi
|
|
71
|
+
sleep 1
|
|
72
|
+
done
|
|
73
|
+
echo "Benchmark server failed to start"
|
|
74
|
+
exit 1
|
|
75
|
+
|
|
76
|
+
- name: Run comparison
|
|
77
|
+
run: |
|
|
78
|
+
mkdir -p compare
|
|
79
|
+
cd compare
|
|
80
|
+
# --refresh: re-check the index so every library resolves to its newest release.
|
|
81
|
+
# The script has no lockfile, so unpinned dependencies float on purpose.
|
|
82
|
+
UV_ARGS=(--refresh)
|
|
83
|
+
BENCH_ARGS=()
|
|
84
|
+
if [ "$HTTPR_SOURCE" = "main" ]; then
|
|
85
|
+
UV_ARGS+=(--with ../dist/httpr-*.whl)
|
|
86
|
+
BENCH_ARGS+=(--httpr-version "${GITHUB_SHA::7}")
|
|
87
|
+
fi
|
|
88
|
+
uv run "${UV_ARGS[@]}" --script ../benchmark/benchmark.py "${BENCH_ARGS[@]}" | tee standard.log
|
|
89
|
+
uv run "${UV_ARGS[@]}" --script ../benchmark/benchmark.py "${BENCH_ARGS[@]}" --run_multithread | tee multithread.log
|
|
90
|
+
|
|
91
|
+
- name: Render chart and tables
|
|
92
|
+
run: |
|
|
93
|
+
uv run --script benchmark/render_comparison.py \
|
|
94
|
+
--results compare --out compare/site \
|
|
95
|
+
--run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
|
96
|
+
|
|
97
|
+
- name: Write job summary
|
|
98
|
+
run: |
|
|
99
|
+
{
|
|
100
|
+
echo "## HTTP-client comparison (httpr from ${HTTPR_SOURCE})"
|
|
101
|
+
echo
|
|
102
|
+
echo "The chart is in the run artifact; the published copy is at https://thomasht86.github.io/httpr/dev/bench/compare/"
|
|
103
|
+
sed '1d' compare/site/comparison.md
|
|
104
|
+
} >> "$GITHUB_STEP_SUMMARY"
|
|
105
|
+
|
|
106
|
+
- name: Upload results
|
|
107
|
+
uses: actions/upload-artifact@v6
|
|
108
|
+
with:
|
|
109
|
+
name: client-comparison-${{ github.run_id }}
|
|
110
|
+
path: |
|
|
111
|
+
compare/*.log
|
|
112
|
+
compare/site/
|
|
113
|
+
|
|
114
|
+
- name: Publish to gh-pages
|
|
115
|
+
if: ${{ github.event_name == 'schedule' || inputs.publish }}
|
|
116
|
+
env:
|
|
117
|
+
GIT_AUTHOR_NAME: github-actions[bot]
|
|
118
|
+
GIT_AUTHOR_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com
|
|
119
|
+
GIT_COMMITTER_NAME: github-actions[bot]
|
|
120
|
+
GIT_COMMITTER_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com
|
|
121
|
+
run: |
|
|
122
|
+
set -euo pipefail
|
|
123
|
+
git clone --depth 1 --branch gh-pages \
|
|
124
|
+
"https://x-access-token:${{ github.token }}@github.com/${GITHUB_REPOSITORY}.git" gh-pages
|
|
125
|
+
STAMP="$(date -u +%Y-%m-%d)"
|
|
126
|
+
publish() {
|
|
127
|
+
DEST=gh-pages/dev/bench/compare
|
|
128
|
+
mkdir -p "$DEST/history/$STAMP"
|
|
129
|
+
cp compare/site/* "$DEST/"
|
|
130
|
+
# Keep a dated copy of the raw numbers so trends can be plotted later.
|
|
131
|
+
cp compare/site/*.csv compare/site/comparison.md "$DEST/history/$STAMP/"
|
|
132
|
+
git -C gh-pages add -A dev/bench/compare
|
|
133
|
+
if git -C gh-pages diff --cached --quiet; then
|
|
134
|
+
echo "No changes to publish"
|
|
135
|
+
return 0
|
|
136
|
+
fi
|
|
137
|
+
git -C gh-pages commit -q -m "Client comparison ${STAMP} (httpr from ${HTTPR_SOURCE})"
|
|
138
|
+
git -C gh-pages push origin gh-pages
|
|
139
|
+
}
|
|
140
|
+
# The docs deploy and the benchmark action also push to gh-pages. On a
|
|
141
|
+
# rejected push, start over from the new tip rather than rebasing a
|
|
142
|
+
# shallow clone.
|
|
143
|
+
for attempt in 1 2 3 4 5; do
|
|
144
|
+
if publish; then
|
|
145
|
+
exit 0
|
|
146
|
+
fi
|
|
147
|
+
echo "push rejected, retrying from the current gh-pages tip (attempt $attempt)"
|
|
148
|
+
sleep 10
|
|
149
|
+
git -C gh-pages fetch --depth 1 origin gh-pages
|
|
150
|
+
git -C gh-pages reset -q --hard FETCH_HEAD
|
|
151
|
+
done
|
|
152
|
+
echo "Could not push to gh-pages after 5 attempts"
|
|
153
|
+
exit 1
|
|
@@ -42,15 +42,19 @@ jobs:
|
|
|
42
42
|
# Fetch gh-pages branch
|
|
43
43
|
git fetch origin gh-pages:gh-pages || true
|
|
44
44
|
|
|
45
|
-
# Save
|
|
45
|
+
# Save everything under dev/bench (httpr trend chart, client comparison)
|
|
46
46
|
mkdir -p /tmp/bench-backup
|
|
47
|
-
if git
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
if git show gh-pages:dev/bench/index.html > /tmp/bench-backup/index.html 2>/dev/null; then
|
|
51
|
-
echo "Found existing benchmark index.html, will preserve it"
|
|
47
|
+
if git rev-parse --verify --quiet gh-pages:dev/bench > /dev/null; then
|
|
48
|
+
git archive gh-pages dev/bench | tar -x -C /tmp/bench-backup
|
|
49
|
+
echo "Preserving dev/bench from gh-pages"
|
|
52
50
|
fi
|
|
53
51
|
|
|
52
|
+
# Drop everything untracked before switching branches: the docs build
|
|
53
|
+
# leaves the compiled extension, __pycache__, .venv and site/ in the
|
|
54
|
+
# tree, and `git add -A` below would otherwise commit them to gh-pages
|
|
55
|
+
# (site/ is already saved in /tmp/site-backup).
|
|
56
|
+
git clean -fdxq
|
|
57
|
+
|
|
54
58
|
# Switch to gh-pages branch
|
|
55
59
|
git checkout gh-pages || git checkout --orphan gh-pages
|
|
56
60
|
|
|
@@ -64,12 +68,11 @@ jobs:
|
|
|
64
68
|
fi
|
|
65
69
|
cp -r /tmp/site-backup/* .
|
|
66
70
|
|
|
67
|
-
# Restore
|
|
68
|
-
if [ -
|
|
69
|
-
mkdir -p dev
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
echo "Restored benchmark chart files"
|
|
71
|
+
# Restore dev/bench
|
|
72
|
+
if [ -d /tmp/bench-backup/dev/bench ]; then
|
|
73
|
+
mkdir -p dev
|
|
74
|
+
cp -r /tmp/bench-backup/dev/bench dev/
|
|
75
|
+
echo "Restored dev/bench"
|
|
73
76
|
fi
|
|
74
77
|
|
|
75
78
|
# Add all files and commit
|
httpr-0.7.1/CLAUDE.md
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
+
|
|
5
|
+
## Project Overview
|
|
6
|
+
|
|
7
|
+
**httpr** is a high-performance HTTP client for Python built in Rust using PyO3 and reqwest. It's designed as a drop-in replacement for `httpx` and `requests` with significantly better performance.
|
|
8
|
+
|
|
9
|
+
## Build & Development Commands
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
# Install development dependencies
|
|
13
|
+
uv sync --extra dev
|
|
14
|
+
|
|
15
|
+
# Build Rust extension (required after any Rust code changes)
|
|
16
|
+
uv run maturin develop
|
|
17
|
+
|
|
18
|
+
# Run tests (depends on httpbin.org)
|
|
19
|
+
uv run pytest tests/
|
|
20
|
+
|
|
21
|
+
# Type checking
|
|
22
|
+
uv run mypy httpr/
|
|
23
|
+
|
|
24
|
+
# Linting
|
|
25
|
+
uv run ruff check httpr/
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
### Pre-commit Hooks
|
|
29
|
+
|
|
30
|
+
Set up pre-commit hooks to automatically lint and format code before commits:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
# Install pre-commit
|
|
34
|
+
uv pip install pre-commit
|
|
35
|
+
|
|
36
|
+
# Install git hooks
|
|
37
|
+
pre-commit install
|
|
38
|
+
pre-commit install --hook-type commit-msg
|
|
39
|
+
|
|
40
|
+
# Run all hooks manually (optional)
|
|
41
|
+
pre-commit run --all-files
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Configured hooks (see `.pre-commit-config.yaml`):
|
|
45
|
+
- **ruff**: Python linting and formatting
|
|
46
|
+
- **mypy**: Python type checking (httpr/ only)
|
|
47
|
+
- **cargo fmt/clippy**: Rust formatting and linting
|
|
48
|
+
- **commitizen/commitlint**: Conventional commit messages
|
|
49
|
+
|
|
50
|
+
### Taskfile Commands
|
|
51
|
+
|
|
52
|
+
This project uses [Taskfile](https://taskfile.dev) for development workflows:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
task --list # List all available tasks
|
|
56
|
+
|
|
57
|
+
# Testing
|
|
58
|
+
task test:unit # Run unit tests only
|
|
59
|
+
task test:e2e # Run e2e tests (requires httpbun running)
|
|
60
|
+
task e2e # Full e2e workflow: certs → start httpbun → test → stop
|
|
61
|
+
task e2e:local # Start httpbun and run e2e tests (keep container running)
|
|
62
|
+
|
|
63
|
+
# Development
|
|
64
|
+
task dev # Build Rust extension
|
|
65
|
+
task check # Run all checks (lint + test) - use before committing
|
|
66
|
+
|
|
67
|
+
# Linting
|
|
68
|
+
task lint # Run Python linters (ruff + mypy)
|
|
69
|
+
task lint:rust # Run Rust linters (fmt + clippy)
|
|
70
|
+
task lint:all # Run all linters (Python + Rust)
|
|
71
|
+
|
|
72
|
+
# Formatting
|
|
73
|
+
task fmt # Format Python code
|
|
74
|
+
task fmt:rust # Format Rust code
|
|
75
|
+
task fmt:all # Format all code
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### E2E Tests with httpbun
|
|
79
|
+
|
|
80
|
+
E2E tests use [httpbun](https://httpbun.com) in Docker with SSL certificates generated by trustme.
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
# One-time setup: add hosts entry
|
|
84
|
+
echo '127.0.0.1 httpbun.local' | sudo tee -a /etc/hosts
|
|
85
|
+
|
|
86
|
+
# Run full e2e workflow
|
|
87
|
+
task e2e
|
|
88
|
+
|
|
89
|
+
# Or run iteratively during development
|
|
90
|
+
task e2e:local # Starts httpbun and runs tests (keeps container running)
|
|
91
|
+
task test:e2e # Run e2e tests again
|
|
92
|
+
task httpbun:stop # Stop container when done
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Key files:
|
|
96
|
+
- `Taskfile.yaml` - Task definitions
|
|
97
|
+
- `scripts/generate_certs.py` - SSL cert generation using trustme
|
|
98
|
+
- `tests/e2e/` - E2E test files
|
|
99
|
+
- `.certs/` - Generated certificates (gitignored)
|
|
100
|
+
|
|
101
|
+
### Benchmarking
|
|
102
|
+
|
|
103
|
+
Comparison against other HTTP clients (published weekly by `.github/workflows/compare.yml` to
|
|
104
|
+
gh-pages `dev/bench/compare/`, rendered by `benchmark/render_comparison.py`, embedded in README and docs):
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
cd benchmark/
|
|
108
|
+
uv run uvicorn server:app # Terminal 1: Start test server
|
|
109
|
+
uv run python benchmark.py # Terminal 2: Run benchmarks
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Regression tracking:
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
task test:benchmark # Wall-clock suite (tests/benchmark/test_performance.py),
|
|
116
|
+
# charted by github-action-benchmark on pushes to main
|
|
117
|
+
task test:codspeed # CodSpeed CPU-simulation suite (tests/benchmark/codspeed/),
|
|
118
|
+
# reported on every PR by .github/workflows/codspeed.yml
|
|
119
|
+
task test:codspeed:wall # Same suite with plain wall-clock timings, no CodSpeed CLI
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Always benchmark a release build (`uv run maturin develop --release`); `task dev`
|
|
123
|
+
builds in debug mode and the numbers are meaningless.
|
|
124
|
+
|
|
125
|
+
## Architecture
|
|
126
|
+
|
|
127
|
+
### Rust Core (`src/`)
|
|
128
|
+
- `lib.rs`: Main `RClient` class with sync request handling via single-threaded Tokio runtime (`LazyLock<Runtime>` with `new_current_thread()`)
|
|
129
|
+
- `request()` method: Buffers entire response body
|
|
130
|
+
- `_stream()` method: Returns `StreamingResponse` without buffering body
|
|
131
|
+
- `response.rs`: Response objects with `CaseInsensitiveHeaderMap` for HTTP/2 compliant header handling
|
|
132
|
+
- `Response`: Standard response with buffered `content`
|
|
133
|
+
- `StreamingResponse`: Holds `Arc<Mutex<Option<reqwest::Response>>>` for chunk iteration
|
|
134
|
+
- `TextIterator`: Iterator for decoding chunks as text
|
|
135
|
+
- `LineIterator`: Iterator for line-by-line reading with internal buffer
|
|
136
|
+
- `traits.rs`: Conversion traits between Python/Rust types (IndexMap ↔ HeaderMap)
|
|
137
|
+
- `utils.rs`: CA certificate loading, encoding detection
|
|
138
|
+
|
|
139
|
+
### Python Wrapper (`httpr/`)
|
|
140
|
+
- `__init__.py`: `Client` (sync) and `AsyncClient` classes with context manager support
|
|
141
|
+
- `stream()` context manager wraps `_stream()` and handles cleanup
|
|
142
|
+
- Both `Client` and `AsyncClient` support streaming; `AsyncClient.stream()` yields an `AsyncStreamingResponse` wrapper with `async for` iteration
|
|
143
|
+
- `close()` / `__exit__` / `aclose()` / `__aexit__` really close the client (see "Client Lifecycle" below)
|
|
144
|
+
- `AsyncClient` uses `asyncio.run_in_executor()` to wrap sync Rust calls - NOT native async
|
|
145
|
+
- `httpr.pyi`: Type stubs for IDE support including `StreamingResponse`, `TextIterator`, `LineIterator`
|
|
146
|
+
|
|
147
|
+
### Key Design Decisions
|
|
148
|
+
1. **Single Tokio Runtime**: All async Rust operations run on one thread
|
|
149
|
+
2. **Async is Sync**: `AsyncClient` runs sync Rust code in thread executor
|
|
150
|
+
3. **Zero Python Dependencies**: All functionality in Rust
|
|
151
|
+
4. **Case-Insensitive Headers**: Custom struct maintains original casing while allowing case-insensitive lookups (HTTP/2 requirement)
|
|
152
|
+
5. **Streaming**: `StreamingResponse` holds reqwest response and provides chunk iteration without buffering entire body
|
|
153
|
+
|
|
154
|
+
## Critical Implementation Details
|
|
155
|
+
|
|
156
|
+
### Python-Rust Interface
|
|
157
|
+
- All Python params converted to strings before passing to Rust
|
|
158
|
+
- HTTP method validation happens in Python wrapper
|
|
159
|
+
- Rust uses `IndexMap<String, String, RandomState>` (foldhash) for dicts
|
|
160
|
+
- Use `Unpack` for `**kwargs` typing (via typing_extensions for Python ≤3.11)
|
|
161
|
+
|
|
162
|
+
### SSL/TLS
|
|
163
|
+
- CA bundle is resolved per client: the `ca_cert_file` param wins, otherwise the `HTTPR_CA_BUNDLE` env var is used as a default
|
|
164
|
+
- The env var is only ever read, never written; `ca_cert_file` is scoped to the client that passed it (see issue #86)
|
|
165
|
+
- mTLS via `client_pem` parameter (file path, PEM format) or `client_pem_data` (bytes, PEM format)
|
|
166
|
+
- `client_pem_data` allows passing certificates without filesystem access (useful for containers/secrets managers)
|
|
167
|
+
- `verify=False` enables `danger_accept_invalid_certs()`
|
|
168
|
+
|
|
169
|
+
### Headers Behavior
|
|
170
|
+
- Headers are lowercased internally (HTTP/2 spec)
|
|
171
|
+
- `client.headers` getter returns a live `_ClientHeaders` view; in-place mutations (`__setitem__`, `del`, `update`, `pop`, `popitem`, `setdefault`, `clear`) mirror to the client via Rust `set_header`/`del_header`
|
|
172
|
+
- `client.headers` getter excludes `Cookie` header; `set_headers` preserves the existing `Cookie` header so header mutation never clobbers cookies
|
|
173
|
+
- `client.cookies` getter/setter extracts from `Cookie` header
|
|
174
|
+
- Per-request `cookies` are merged into the client's `Cookie` header (request wins per name) so exactly one `Cookie` header is sent (RFC 6265 §5.4; issue #82). Do not append a second header
|
|
175
|
+
|
|
176
|
+
### Query Parameters (issues #82, #87)
|
|
177
|
+
- `params` (client-level and per-request) and form `data` are normalised in Rust (`src/params.rs`) into ordered `(key, value)` pairs with httpx rules: `True`/`False` → `true`/`false`, `None` → empty, list/tuple → repeated key, else `str()`. Accepts a mapping or a sequence of pairs. There is no Python-side `str(v)` coercion any more; do not re-add one
|
|
178
|
+
- Client params are merged with request params: client pairs first, then the request's; a key the request supplies replaces the client's values for it. `client.params` returns a dict whose values are `str` or `list[str]` for repeated keys and round-trips through the setter
|
|
179
|
+
- Request building (`build_request`, `send_request`, `response_meta` in `src/lib.rs`) is shared by `request()` and `_stream()`; change it once
|
|
180
|
+
|
|
181
|
+
### Request Body
|
|
182
|
+
- Mutually exclusive: `content` (bytes), `data` (form), `json` (JSON), `files` (multipart)
|
|
183
|
+
- A body is sent for any method the caller supplies one for, including GET/DELETE/OPTIONS (RFC 9110, matches httpx; issue #83). Do not re-add a POST/PUT/PATCH guard
|
|
184
|
+
- `json` uses `pythonize::depythonize()`; `data` goes through `normalize_params` so list values become repeated fields and insertion order is preserved (serde_json would alphabetise)
|
|
185
|
+
- `files` dict maps field names to file paths
|
|
186
|
+
|
|
187
|
+
### Proxy
|
|
188
|
+
- Set via `proxy` param or `HTTPR_PROXY` env var; the env var is read once in `new()`, never in the setter
|
|
189
|
+
- Changing `client.proxy` rebuilds the entire reqwest client (expensive). `RClient.config: ClientConfig` keeps the construction settings (loaded root certs, mTLS `Identity`, redirects, verify, https_only, http2_only, cookie_store, referer) and `ClientConfig::build()` is the single place a `reqwest::Client` is built, used by `new()` and `set_proxy()`; add any new builder setting there, not inline (issue #84). The setter takes `Option<String>`; `None` removes the proxy. The rebuilt client starts with an empty cookie store
|
|
190
|
+
|
|
191
|
+
### Client Lifecycle (issue #88)
|
|
192
|
+
- All lifecycle state lives in `src/lifecycle.rs`: `ClientState` (`Arc`-shared between an `RClient` and its in-flight requests) holds `client: Mutex<Option<reqwest::Client>>`, an `in_flight` counter and a `CancellationToken` for pending connects. `RClient::close()` takes the client (`None`); dropping the `reqwest::Client` drops the client's handle to its connection pool
|
|
193
|
+
- Connection tasks live on the single-threaded `RUNTIME` and only progress inside `block_on`, so dropping a pool does not close sockets by itself: `settle_dropped_pool()` runs a fixed number of `yield_now` turns so the connection tasks observe the hang-up and send FIN (deterministic, never waits on I/O). Each turn is a driver-poll syscall and this runs on every close/drop, so keep `SETTLE_SCHEDULER_TURNS` small: 4 costs ~40 µs, 32 doubled the create-request-close benchmark (`test_single_request`). The first turn already drains the whole wake-up chain; the rest are margin. If another thread holds the scheduler core (it is mid-request), its scheduler closes them instead
|
|
194
|
+
- Overlapping requests make hyper race a fresh connect against the idle-pool checkout; a losing connect is finished in the background and holds the pool alive until it resolves, which needs real I/O and against a remote host can take longer than `close()` should wait. So every `reqwest::Client` is built with `CancelConnectsLayer` (a tower layer over reqwest's connector, via `ClientBuilder::connector_layer`) and `close()` cancels the token when nothing is in flight; cancelled connects resolve on the next scheduler turn and drop their pool handle. Do not replace this with waiting or with `num_alive_tasks` heuristics: the count is runtime-global and a pending connect keeps it constant
|
|
195
|
+
- `begin_request()` increments `in_flight` before cloning the `reqwest::Client`, so a request already in flight keeps the pool alive and finishes normally when the client is closed underneath it. While anything is in flight `close()` only drops its own handle and yields; the `InFlight` guard of the last request to finish (in `request()` after the body is buffered, or `StreamingResponse::close()`/drop for streams) cancels pending connects and settles. `StreamingResponse` owns its `InFlight` for its whole lifetime, since its connection stays busy until it is closed
|
|
196
|
+
- `set_proxy` swaps the client via `ClientState::replace()` and settles the old pool; it raises `ClientClosed` on a closed client. `RClient` implements `Drop`, so a client garbage-collected without `close()` releases its pool as well (safe: every `block_on` in the crate runs with the GIL released, so dealloc never happens inside one)
|
|
197
|
+
- Use after close raises `ClientClosed`, a `RuntimeError` subclass (httpx raises plain `RuntimeError` here); header/cookie getters keep working. `is_closed` getter mirrors httpx. The message lives once in Rust (`CLIENT_CLOSED_MSG`, exported as `_CLIENT_CLOSED_MSG`)
|
|
198
|
+
- Python: `Client.close()`/`__exit__` call the Rust `close()`; `AsyncClient.aclose()`/`__aexit__` additionally `shutdown(wait=False)` the client's own `ThreadPoolExecutor`, and run on the event-loop thread on purpose (sub-millisecond, no I/O wait). `_run_sync_asyncio` checks `is_closed` first and also maps the executor's "cannot schedule new futures after shutdown" `RuntimeError` to `ClientClosed`, which covers a `close()` racing in from another OS thread
|
|
199
|
+
- Leaving a `with`/`async with` block closes the client; re-entering it afterwards raises `ClientClosed` on the next request. pyvespa's `VespaSync`/`VespaAsync` given an external session never close it and are unaffected; with an owned client they close but never null it, so re-entering the same wrapper object raises `ClientClosed` (see memory note; needs a pyvespa-side fix)
|
|
200
|
+
|
|
201
|
+
### Timeouts (issue #81)
|
|
202
|
+
- Default `timeout` is 30 s and lives in the PyO3 `#[new]` signature in `src/lib.rs`; `Client.__init__` in Python only documents the parameters and forwards nothing, so its defaults must match Rust's (`test_python_signature_matches_rust_defaults` enforces this). `None` disables the timeout
|
|
203
|
+
- The timeout is an inactivity bound, not a total deadline: `src/timeout.rs` wraps the wait for the response headers (`send_request`) and the wait for each body chunk (`read_body` for buffered responses, `next_chunk` for `StreamingResponse` and its iterators) in `tokio::time::timeout`. A stream that keeps delivering data is never cut off, matching httpx's read timeout. Both phases raise `ReadTimeout` (via the `TimedOut` error type, mapped first in `map_anyhow_error`)
|
|
204
|
+
- Do not use `reqwest::ClientBuilder::timeout` or `RequestBuilder::timeout`: the former is baked into the built client so `client.timeout = ...` could not change it, and both are total deadlines that also kill a live streaming body. `RClient.timeout` is a plain field read at request time in `build_request`
|
|
205
|
+
- `StreamingResponse`, `TextIterator` and `LineIterator` carry the `Option<Duration>` they were created with
|
|
206
|
+
|
|
207
|
+
### Streaming Responses
|
|
208
|
+
- `_stream()` method returns `StreamingResponse` without calling `.bytes()` on reqwest response
|
|
209
|
+
- `StreamingResponse` holds `Arc<Mutex<Option<reqwest::Response>>>` to allow chunk reading across Python GIL boundaries
|
|
210
|
+
- Chunk iteration uses `RUNTIME.block_on()` with `py.allow_threads()` to read each chunk
|
|
211
|
+
- State tracking via `Arc<Mutex<bool>>` for `closed` and `consumed` flags
|
|
212
|
+
- Three iteration modes:
|
|
213
|
+
- `iter_bytes()`: Direct chunk iteration (returns `Iterator[bytes]`)
|
|
214
|
+
- `iter_text()`: Returns `TextIterator` that decodes chunks using response encoding
|
|
215
|
+
- `iter_lines()`: Returns `LineIterator` with internal buffer for line-by-line reading
|
|
216
|
+
- `read()` method consumes remaining response body and marks as consumed
|
|
217
|
+
- `close()` method sets closed flag and drops the response
|
|
218
|
+
- Python wrapper uses `@contextmanager` to ensure `close()` is called on exit
|
|
219
|
+
- AsyncClient streaming (issue #85): `AsyncClient.stream()` yields `AsyncStreamingResponse` (pure Python, `httpr/__init__.py`), which wraps the Rust `StreamingResponse` and adds `aiter_bytes()`/`aiter_text()`/`aiter_lines()`/`aread()`/`__aiter__`. Each step runs `next()` on the underlying sync iterator via `_run_sync_asyncio` on the client's executor so the event loop is never blocked; after `aclose()` the next step raises `ClientClosed`. The sync `iter_*`/`read()`/`__iter__` are kept on the wrapper and delegate directly (they block the loop). The Rust `StreamingResponse` is not subclassable, hence the wrapper rather than a subclass
|
|
220
|
+
|
|
221
|
+
## What NOT to Do
|
|
222
|
+
|
|
223
|
+
- Don't add Python dependencies (defeats "zero dependencies" goal)
|
|
224
|
+
- Don't use native async Rust in request path (breaks single-threaded runtime model)
|
|
225
|
+
- Don't modify headers case-sensitivity behavior (HTTP/2 spec requirement)
|
|
226
|
+
- Don't skip `maturin develop` after Rust changes
|