httpie-hmac 1.2.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- httpie_hmac-2.0.0/PKG-INFO +207 -0
- httpie_hmac-2.0.0/README.md +188 -0
- httpie_hmac-2.0.0/httpie_hmac/_body.py +215 -0
- httpie_hmac-2.0.0/httpie_hmac/_s3_auth.py +137 -0
- {httpie-hmac-1.2.0 → httpie_hmac-2.0.0}/httpie_hmac/httpie_hmac.py +99 -41
- httpie_hmac-2.0.0/httpie_hmac.egg-info/PKG-INFO +207 -0
- httpie_hmac-2.0.0/httpie_hmac.egg-info/SOURCES.txt +26 -0
- {httpie-hmac-1.2.0 → httpie_hmac-2.0.0}/pyproject.toml +1 -1
- httpie_hmac-2.0.0/tests/test_aws_endpoints.py +101 -0
- httpie_hmac-2.0.0/tests/test_aws_session_token.py +62 -0
- httpie_hmac-2.0.0/tests/test_aws_signature.py +54 -0
- httpie_hmac-2.0.0/tests/test_compression_limitation.py +84 -0
- httpie_hmac-2.0.0/tests/test_configuration.py +96 -0
- httpie_hmac-2.0.0/tests/test_custom_format.py +60 -0
- httpie_hmac-2.0.0/tests/test_header_types.py +88 -0
- httpie_hmac-2.0.0/tests/test_httpie_cli.py +80 -0
- httpie_hmac-2.0.0/tests/test_s3_headers.py +130 -0
- httpie_hmac-2.0.0/tests/test_s3_paths.py +144 -0
- httpie_hmac-2.0.0/tests/test_simple.py +89 -0
- httpie_hmac-2.0.0/tests/test_upload_bodies.py +207 -0
- httpie_hmac-2.0.0/tests/test_upload_limits.py +345 -0
- httpie-hmac-1.2.0/PKG-INFO +0 -104
- httpie-hmac-1.2.0/README.md +0 -89
- httpie-hmac-1.2.0/httpie_hmac.egg-info/PKG-INFO +0 -104
- httpie-hmac-1.2.0/httpie_hmac.egg-info/SOURCES.txt +0 -11
- {httpie-hmac-1.2.0 → httpie_hmac-2.0.0}/LICENSE +0 -0
- {httpie-hmac-1.2.0 → httpie_hmac-2.0.0}/httpie_hmac/__init__.py +0 -0
- {httpie-hmac-1.2.0 → httpie_hmac-2.0.0}/httpie_hmac.egg-info/dependency_links.txt +0 -0
- {httpie-hmac-1.2.0 → httpie_hmac-2.0.0}/httpie_hmac.egg-info/entry_points.txt +0 -0
- {httpie-hmac-1.2.0 → httpie_hmac-2.0.0}/httpie_hmac.egg-info/requires.txt +0 -0
- {httpie-hmac-1.2.0 → httpie_hmac-2.0.0}/httpie_hmac.egg-info/top_level.txt +0 -0
- {httpie-hmac-1.2.0 → httpie_hmac-2.0.0}/setup.cfg +0 -0
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: httpie-hmac
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: HMAC Auth Plugin for Httpie
|
|
5
|
+
Author-email: Martyn Pittuck-Schols <martyn@rustfoo.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: homepage, https://github.com/martynp/httpie-hmac/
|
|
8
|
+
Project-URL: repository, https://github.com/martynp/httpie-hmac.git
|
|
9
|
+
Keywords: httpie,auth,hmac,aws4
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Requires-Python: >=3.7
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: aws_requests_auth
|
|
16
|
+
Requires-Dist: httpie
|
|
17
|
+
Requires-Dist: requests
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# HTTPIE HMAC
|
|
21
|
+
|
|
22
|
+
For upgrades from version 1.x to 2.0.0, follow the [migration guide](MIGRATION.md).
|
|
23
|
+
|
|
24
|
+
This plugin is borrows heavily from the original work by Nick Statterly (https://github.com/guardian/httpie-hmac-auth) - that project was archived in May 2022.
|
|
25
|
+
|
|
26
|
+
This plugin extends the functionality to allow different HMAC patterns to be defined in the library and by a user provided script - thereby avoiding any requirement to create a new plugin to support a different pattern.
|
|
27
|
+
|
|
28
|
+
The httpie auth should be set to ``hmac`` and the ``--auth`` field contains key-value pairs to configure the plugin, the keys are:
|
|
29
|
+
|
|
30
|
+
* ``secret`` - base64 encoded secret to be used in the HMAC
|
|
31
|
+
* ``access_id`` - (Optional) String access token / id used to identify the user depending on the schema
|
|
32
|
+
* ``format`` - (Optional) Sets a pre-defined format or a python file to process the headers
|
|
33
|
+
* ``session_token`` - (Optional, AWS4 only) Session token for temporary AWS credentials
|
|
34
|
+
* ``max_body_size`` - (Optional) Maximum upload size in bytes; defaults to 104857600 (100 MiB)
|
|
35
|
+
|
|
36
|
+
Key-value pairs can also be set using environment variables starting with `HTTPIE_HMAC_`.
|
|
37
|
+
|
|
38
|
+
The first colon separates each key from its value; additional colons are kept
|
|
39
|
+
in the value, so `host:localhost:9000` is supported. Commas separate settings.
|
|
40
|
+
|
|
41
|
+
For example:
|
|
42
|
+
|
|
43
|
+
``` bash
|
|
44
|
+
http --auth-type=hmac --auth="secret:some_secret" GET http://localhost:8000
|
|
45
|
+
http --auth-type=hmac --auth="secret:7Ez...wVA,access_id:AK...6R,format:aws4" GET https://my_bucket.s3.eu-west-2.amazonaws.com/file.txt
|
|
46
|
+
|
|
47
|
+
export HTTPIE_HMAC_SECRET=7Ez...wVA
|
|
48
|
+
export HTTPIE_HMAC_ACCESS_ID=AK...6R
|
|
49
|
+
export HTTPIE_HMAC_FORMAT=aws4
|
|
50
|
+
httpie --auth-type=hmac --auth="" GET https://my_bucket.s3.eu-west-2.amazonaws.com/file.txt
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Supported Formats
|
|
54
|
+
|
|
55
|
+
### AWS4 (aws4)
|
|
56
|
+
|
|
57
|
+
AWS4 uses the `AWSRequestsAuth` library to generate the required AWS auth header. It will attempt to get the required information from the provided URL, however the host, region and service fields can be set manually.
|
|
58
|
+
|
|
59
|
+
Explicit `region` and `service` settings each override the inferred value.
|
|
60
|
+
Inference supports `service.region.amazonaws.com`, regional S3 bucket endpoints
|
|
61
|
+
(including dualstack), API Gateway API endpoints, and OpenSearch domain
|
|
62
|
+
endpoints. Other layouts require explicit `region` and `service` settings.
|
|
63
|
+
Hostname service names that differ from the AWS signing service name also need
|
|
64
|
+
an explicit `service` setting. The `host` setting overrides the signing host;
|
|
65
|
+
inference still uses the request URL.
|
|
66
|
+
|
|
67
|
+
For example:
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
http --auth-type=hmac --auth="secret:7Ez...wVA,access_id:AK...6R,host:my_bucket.s3.eu-west-2.amazonaws.com,service:s3,region:eu-west-2,format:aws4" GET https://my_bucket.s3.eu-west-2.amazonaws.com/file.txt
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
For temporary AWS credentials (such as credentials obtained through STS or an
|
|
74
|
+
assumed role), supply the session token alongside the matching access key ID and
|
|
75
|
+
secret access key. The AWS4 signer includes it as `X-Amz-Security-Token`:
|
|
76
|
+
|
|
77
|
+
``` bash
|
|
78
|
+
export HTTPIE_HMAC_ACCESS_ID="$AWS_ACCESS_KEY_ID"
|
|
79
|
+
export HTTPIE_HMAC_SECRET="$AWS_SECRET_ACCESS_KEY"
|
|
80
|
+
export HTTPIE_HMAC_SESSION_TOKEN="$AWS_SESSION_TOKEN"
|
|
81
|
+
http --auth-type=hmac --auth="format:aws4" GET https://my-bucket.s3.eu-west-2.amazonaws.com/file.txt
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
You can also pass `session_token:YOUR_SESSION_TOKEN` in the comma-separated
|
|
85
|
+
`--auth` settings. Command-line settings override the corresponding
|
|
86
|
+
`HTTPIE_HMAC_` environment variables. Supply the AWS secret access key as issued,
|
|
87
|
+
without decoding it. Credentials must be supplied explicitly; this plugin does
|
|
88
|
+
not discover AWS profiles or refresh expired credentials. When switching to
|
|
89
|
+
long-lived credentials, unset `HTTPIE_HMAC_SESSION_TOKEN`.
|
|
90
|
+
|
|
91
|
+
### Simple (simple)
|
|
92
|
+
|
|
93
|
+
The string_to_sign consists of the HTTP method, content_md5, content_type, http_date and path:
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
[method]\n
|
|
97
|
+
[content_md5]\n
|
|
98
|
+
[content_type]\n
|
|
99
|
+
[http_date]\n
|
|
100
|
+
[path]
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
This string is signed using the sha256 HMAC. The resulting signature is placed in the "Authorization" header in the format:
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
Authorization: HMAC [signature]
|
|
107
|
+
Authorization: HMAC [access_id]:[signature]
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Custom Format
|
|
111
|
+
|
|
112
|
+
A custom python file can be passed to the plug and used to generate bespoke formats, the following example implements the Simple formatter using a custom file:
|
|
113
|
+
|
|
114
|
+
```
|
|
115
|
+
import hmac
|
|
116
|
+
import hashlib
|
|
117
|
+
import base64
|
|
118
|
+
|
|
119
|
+
from httpie_hmac import HmacGenerate
|
|
120
|
+
|
|
121
|
+
class HmacAuthCustom(HmacGenerate):
|
|
122
|
+
|
|
123
|
+
def generate(request):
|
|
124
|
+
|
|
125
|
+
string_to_sign = '\n'.join(
|
|
126
|
+
[request.method, request.content_md5, request.content_type,
|
|
127
|
+
request.http_date, request.path]).encode()
|
|
128
|
+
digest = hmac.new(bytes(request.secret_key, 'UTF-8'), string_to_sign,
|
|
129
|
+
hashlib.sha256).digest()
|
|
130
|
+
signature = base64.b64encode(digest).rstrip().decode('utf-8')
|
|
131
|
+
|
|
132
|
+
if request.access_id is None or request.access_id == '':
|
|
133
|
+
request.inner.headers['Authorization'] = f"HMAC {signature}"
|
|
134
|
+
else:
|
|
135
|
+
request.inner.headers['Authorization'] = \
|
|
136
|
+
f"HMAC {request.access_id}:{signature}"
|
|
137
|
+
|
|
138
|
+
return request.inner
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Note that the ``request.inner.headers`` dictionary will contain `content_type`, `content_md5` and `date` fields if they were not previously set. If they are not required they need to be removed from the list.
|
|
142
|
+
|
|
143
|
+
Additional data could be passed to the custom formatter using environment variables if needed.
|
|
144
|
+
|
|
145
|
+
## Upload preparation and limits
|
|
146
|
+
|
|
147
|
+
Uploads are hashed before transmission, including multipart and chunked bodies.
|
|
148
|
+
Seekable binary inputs are read in 64 KiB chunks and rewound to their original
|
|
149
|
+
position. Nonseekable and text inputs use a temporary spool, which holds up to
|
|
150
|
+
1 MiB of payload in memory before rolling over to disk. Chunked framing is
|
|
151
|
+
preserved.
|
|
152
|
+
|
|
153
|
+
Every body has a default maximum of **100 MiB** (104857600 bytes). Set
|
|
154
|
+
`max_body_size:209715200` in `--auth`, or export
|
|
155
|
+
`HTTPIE_HMAC_MAX_BODY_SIZE=209715200`, to allow up to 200 MiB. CLI settings take
|
|
156
|
+
precedence. The limit must be a positive integer; unlimited values are not
|
|
157
|
+
accepted. It counts actual payload bytes, including UTF-8 encoding, independently
|
|
158
|
+
of a supplied `Content-Length`. Preparation stops with an error when the limit
|
|
159
|
+
is exceeded, before transmission. This counts payload bytes, excluding wire
|
|
160
|
+
chunk-framing overhead, and bounds spool disk usage per request. Concurrent
|
|
161
|
+
uploads have separate limits. It does not bound allocations made by HTTPie or a
|
|
162
|
+
custom input producer before this plugin receives the body, or time spent
|
|
163
|
+
waiting for a blocked input source.
|
|
164
|
+
|
|
165
|
+
Custom formatters should support a seekable file-like `request.inner.body` as
|
|
166
|
+
well as bytes. Reading it must preserve/restore its position before returning.
|
|
167
|
+
Small nonseekable inputs are converted to bytes; larger spools remain file-like.
|
|
168
|
+
Caller-owned streams are never closed by the plugin.
|
|
169
|
+
|
|
170
|
+
Temporary files close on preparation/signing errors and on the final HTTP
|
|
171
|
+
response; redirects retain the body for replay. Python callers should also close
|
|
172
|
+
a file-like prepared body in a `finally` block after sending or previewing it,
|
|
173
|
+
especially on network errors. HTTPie's auth API has no offline-completion or
|
|
174
|
+
network-error cleanup hook, so offline/abandoned requests additionally rely on
|
|
175
|
+
object cleanup to release their spool. A closed body cannot be replayed.
|
|
176
|
+
HTTPie's request-body output callbacks can run during preparation; spooled
|
|
177
|
+
uploads may not appear in verbose request-body output.
|
|
178
|
+
|
|
179
|
+
## Known limitations
|
|
180
|
+
|
|
181
|
+
Do not combine HMAC authentication with HTTPie's `--compress` option (including
|
|
182
|
+
repeated `--compress`). HTTPie applies request compression after this plugin
|
|
183
|
+
calculates its checksums and signature. If the body is compressed, the generated
|
|
184
|
+
`Content-MD5` no longer matches the payload; AWS4's `x-amz-content-sha256` and
|
|
185
|
+
signature also describe the original body. This affects Simple, AWS4, and custom
|
|
186
|
+
formats that sign the body or its checksums, including offline request previews.
|
|
187
|
+
The plugin does not detect or reject this combination. Omit `--compress` when
|
|
188
|
+
using HMAC authentication. Supporting it requires an HTTPie authentication hook
|
|
189
|
+
after request transformations and before preview or transmission.
|
|
190
|
+
|
|
191
|
+
## Testing
|
|
192
|
+
|
|
193
|
+
Install the package in a virtual environment before running the suite so HTTPie
|
|
194
|
+
can discover the authentication plugin through its installed entry point:
|
|
195
|
+
|
|
196
|
+
``` bash
|
|
197
|
+
python -m venv .venv
|
|
198
|
+
source .venv/bin/activate
|
|
199
|
+
python -m pip install .
|
|
200
|
+
python -m unittest discover -s tests -v
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
The tests use dummy credentials and run offline, including the HTTPie CLI tests.
|
|
204
|
+
They require no AWS account or secrets. CI runs the same suite on Python 3.11
|
|
205
|
+
and 3.13; the existing live AWS smoke test is a separate job.
|
|
206
|
+
Compression regression tests are marked as expected failures until HTTPie
|
|
207
|
+
provides the integration needed to sign the final payload.
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# HTTPIE HMAC
|
|
2
|
+
|
|
3
|
+
For upgrades from version 1.x to 2.0.0, follow the [migration guide](MIGRATION.md).
|
|
4
|
+
|
|
5
|
+
This plugin is borrows heavily from the original work by Nick Statterly (https://github.com/guardian/httpie-hmac-auth) - that project was archived in May 2022.
|
|
6
|
+
|
|
7
|
+
This plugin extends the functionality to allow different HMAC patterns to be defined in the library and by a user provided script - thereby avoiding any requirement to create a new plugin to support a different pattern.
|
|
8
|
+
|
|
9
|
+
The httpie auth should be set to ``hmac`` and the ``--auth`` field contains key-value pairs to configure the plugin, the keys are:
|
|
10
|
+
|
|
11
|
+
* ``secret`` - base64 encoded secret to be used in the HMAC
|
|
12
|
+
* ``access_id`` - (Optional) String access token / id used to identify the user depending on the schema
|
|
13
|
+
* ``format`` - (Optional) Sets a pre-defined format or a python file to process the headers
|
|
14
|
+
* ``session_token`` - (Optional, AWS4 only) Session token for temporary AWS credentials
|
|
15
|
+
* ``max_body_size`` - (Optional) Maximum upload size in bytes; defaults to 104857600 (100 MiB)
|
|
16
|
+
|
|
17
|
+
Key-value pairs can also be set using environment variables starting with `HTTPIE_HMAC_`.
|
|
18
|
+
|
|
19
|
+
The first colon separates each key from its value; additional colons are kept
|
|
20
|
+
in the value, so `host:localhost:9000` is supported. Commas separate settings.
|
|
21
|
+
|
|
22
|
+
For example:
|
|
23
|
+
|
|
24
|
+
``` bash
|
|
25
|
+
http --auth-type=hmac --auth="secret:some_secret" GET http://localhost:8000
|
|
26
|
+
http --auth-type=hmac --auth="secret:7Ez...wVA,access_id:AK...6R,format:aws4" GET https://my_bucket.s3.eu-west-2.amazonaws.com/file.txt
|
|
27
|
+
|
|
28
|
+
export HTTPIE_HMAC_SECRET=7Ez...wVA
|
|
29
|
+
export HTTPIE_HMAC_ACCESS_ID=AK...6R
|
|
30
|
+
export HTTPIE_HMAC_FORMAT=aws4
|
|
31
|
+
httpie --auth-type=hmac --auth="" GET https://my_bucket.s3.eu-west-2.amazonaws.com/file.txt
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Supported Formats
|
|
35
|
+
|
|
36
|
+
### AWS4 (aws4)
|
|
37
|
+
|
|
38
|
+
AWS4 uses the `AWSRequestsAuth` library to generate the required AWS auth header. It will attempt to get the required information from the provided URL, however the host, region and service fields can be set manually.
|
|
39
|
+
|
|
40
|
+
Explicit `region` and `service` settings each override the inferred value.
|
|
41
|
+
Inference supports `service.region.amazonaws.com`, regional S3 bucket endpoints
|
|
42
|
+
(including dualstack), API Gateway API endpoints, and OpenSearch domain
|
|
43
|
+
endpoints. Other layouts require explicit `region` and `service` settings.
|
|
44
|
+
Hostname service names that differ from the AWS signing service name also need
|
|
45
|
+
an explicit `service` setting. The `host` setting overrides the signing host;
|
|
46
|
+
inference still uses the request URL.
|
|
47
|
+
|
|
48
|
+
For example:
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
http --auth-type=hmac --auth="secret:7Ez...wVA,access_id:AK...6R,host:my_bucket.s3.eu-west-2.amazonaws.com,service:s3,region:eu-west-2,format:aws4" GET https://my_bucket.s3.eu-west-2.amazonaws.com/file.txt
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
For temporary AWS credentials (such as credentials obtained through STS or an
|
|
55
|
+
assumed role), supply the session token alongside the matching access key ID and
|
|
56
|
+
secret access key. The AWS4 signer includes it as `X-Amz-Security-Token`:
|
|
57
|
+
|
|
58
|
+
``` bash
|
|
59
|
+
export HTTPIE_HMAC_ACCESS_ID="$AWS_ACCESS_KEY_ID"
|
|
60
|
+
export HTTPIE_HMAC_SECRET="$AWS_SECRET_ACCESS_KEY"
|
|
61
|
+
export HTTPIE_HMAC_SESSION_TOKEN="$AWS_SESSION_TOKEN"
|
|
62
|
+
http --auth-type=hmac --auth="format:aws4" GET https://my-bucket.s3.eu-west-2.amazonaws.com/file.txt
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
You can also pass `session_token:YOUR_SESSION_TOKEN` in the comma-separated
|
|
66
|
+
`--auth` settings. Command-line settings override the corresponding
|
|
67
|
+
`HTTPIE_HMAC_` environment variables. Supply the AWS secret access key as issued,
|
|
68
|
+
without decoding it. Credentials must be supplied explicitly; this plugin does
|
|
69
|
+
not discover AWS profiles or refresh expired credentials. When switching to
|
|
70
|
+
long-lived credentials, unset `HTTPIE_HMAC_SESSION_TOKEN`.
|
|
71
|
+
|
|
72
|
+
### Simple (simple)
|
|
73
|
+
|
|
74
|
+
The string_to_sign consists of the HTTP method, content_md5, content_type, http_date and path:
|
|
75
|
+
|
|
76
|
+
```
|
|
77
|
+
[method]\n
|
|
78
|
+
[content_md5]\n
|
|
79
|
+
[content_type]\n
|
|
80
|
+
[http_date]\n
|
|
81
|
+
[path]
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
This string is signed using the sha256 HMAC. The resulting signature is placed in the "Authorization" header in the format:
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
Authorization: HMAC [signature]
|
|
88
|
+
Authorization: HMAC [access_id]:[signature]
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Custom Format
|
|
92
|
+
|
|
93
|
+
A custom python file can be passed to the plug and used to generate bespoke formats, the following example implements the Simple formatter using a custom file:
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
import hmac
|
|
97
|
+
import hashlib
|
|
98
|
+
import base64
|
|
99
|
+
|
|
100
|
+
from httpie_hmac import HmacGenerate
|
|
101
|
+
|
|
102
|
+
class HmacAuthCustom(HmacGenerate):
|
|
103
|
+
|
|
104
|
+
def generate(request):
|
|
105
|
+
|
|
106
|
+
string_to_sign = '\n'.join(
|
|
107
|
+
[request.method, request.content_md5, request.content_type,
|
|
108
|
+
request.http_date, request.path]).encode()
|
|
109
|
+
digest = hmac.new(bytes(request.secret_key, 'UTF-8'), string_to_sign,
|
|
110
|
+
hashlib.sha256).digest()
|
|
111
|
+
signature = base64.b64encode(digest).rstrip().decode('utf-8')
|
|
112
|
+
|
|
113
|
+
if request.access_id is None or request.access_id == '':
|
|
114
|
+
request.inner.headers['Authorization'] = f"HMAC {signature}"
|
|
115
|
+
else:
|
|
116
|
+
request.inner.headers['Authorization'] = \
|
|
117
|
+
f"HMAC {request.access_id}:{signature}"
|
|
118
|
+
|
|
119
|
+
return request.inner
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Note that the ``request.inner.headers`` dictionary will contain `content_type`, `content_md5` and `date` fields if they were not previously set. If they are not required they need to be removed from the list.
|
|
123
|
+
|
|
124
|
+
Additional data could be passed to the custom formatter using environment variables if needed.
|
|
125
|
+
|
|
126
|
+
## Upload preparation and limits
|
|
127
|
+
|
|
128
|
+
Uploads are hashed before transmission, including multipart and chunked bodies.
|
|
129
|
+
Seekable binary inputs are read in 64 KiB chunks and rewound to their original
|
|
130
|
+
position. Nonseekable and text inputs use a temporary spool, which holds up to
|
|
131
|
+
1 MiB of payload in memory before rolling over to disk. Chunked framing is
|
|
132
|
+
preserved.
|
|
133
|
+
|
|
134
|
+
Every body has a default maximum of **100 MiB** (104857600 bytes). Set
|
|
135
|
+
`max_body_size:209715200` in `--auth`, or export
|
|
136
|
+
`HTTPIE_HMAC_MAX_BODY_SIZE=209715200`, to allow up to 200 MiB. CLI settings take
|
|
137
|
+
precedence. The limit must be a positive integer; unlimited values are not
|
|
138
|
+
accepted. It counts actual payload bytes, including UTF-8 encoding, independently
|
|
139
|
+
of a supplied `Content-Length`. Preparation stops with an error when the limit
|
|
140
|
+
is exceeded, before transmission. This counts payload bytes, excluding wire
|
|
141
|
+
chunk-framing overhead, and bounds spool disk usage per request. Concurrent
|
|
142
|
+
uploads have separate limits. It does not bound allocations made by HTTPie or a
|
|
143
|
+
custom input producer before this plugin receives the body, or time spent
|
|
144
|
+
waiting for a blocked input source.
|
|
145
|
+
|
|
146
|
+
Custom formatters should support a seekable file-like `request.inner.body` as
|
|
147
|
+
well as bytes. Reading it must preserve/restore its position before returning.
|
|
148
|
+
Small nonseekable inputs are converted to bytes; larger spools remain file-like.
|
|
149
|
+
Caller-owned streams are never closed by the plugin.
|
|
150
|
+
|
|
151
|
+
Temporary files close on preparation/signing errors and on the final HTTP
|
|
152
|
+
response; redirects retain the body for replay. Python callers should also close
|
|
153
|
+
a file-like prepared body in a `finally` block after sending or previewing it,
|
|
154
|
+
especially on network errors. HTTPie's auth API has no offline-completion or
|
|
155
|
+
network-error cleanup hook, so offline/abandoned requests additionally rely on
|
|
156
|
+
object cleanup to release their spool. A closed body cannot be replayed.
|
|
157
|
+
HTTPie's request-body output callbacks can run during preparation; spooled
|
|
158
|
+
uploads may not appear in verbose request-body output.
|
|
159
|
+
|
|
160
|
+
## Known limitations
|
|
161
|
+
|
|
162
|
+
Do not combine HMAC authentication with HTTPie's `--compress` option (including
|
|
163
|
+
repeated `--compress`). HTTPie applies request compression after this plugin
|
|
164
|
+
calculates its checksums and signature. If the body is compressed, the generated
|
|
165
|
+
`Content-MD5` no longer matches the payload; AWS4's `x-amz-content-sha256` and
|
|
166
|
+
signature also describe the original body. This affects Simple, AWS4, and custom
|
|
167
|
+
formats that sign the body or its checksums, including offline request previews.
|
|
168
|
+
The plugin does not detect or reject this combination. Omit `--compress` when
|
|
169
|
+
using HMAC authentication. Supporting it requires an HTTPie authentication hook
|
|
170
|
+
after request transformations and before preview or transmission.
|
|
171
|
+
|
|
172
|
+
## Testing
|
|
173
|
+
|
|
174
|
+
Install the package in a virtual environment before running the suite so HTTPie
|
|
175
|
+
can discover the authentication plugin through its installed entry point:
|
|
176
|
+
|
|
177
|
+
``` bash
|
|
178
|
+
python -m venv .venv
|
|
179
|
+
source .venv/bin/activate
|
|
180
|
+
python -m pip install .
|
|
181
|
+
python -m unittest discover -s tests -v
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
The tests use dummy credentials and run offline, including the HTTPie CLI tests.
|
|
185
|
+
They require no AWS account or secrets. CI runs the same suite on Python 3.11
|
|
186
|
+
and 3.13; the existing live AWS smoke test is a separate job.
|
|
187
|
+
Compression regression tests are marked as expected failures until HTTPie
|
|
188
|
+
provides the integration needed to sign the final payload.
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Bounded upload preparation shared by the HMAC formatters."""
|
|
2
|
+
import base64
|
|
3
|
+
import hashlib
|
|
4
|
+
import tempfile
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
CHUNK_SIZE = 64 * 1024
|
|
8
|
+
SPOOL_MEMORY_SIZE = 1024 * 1024
|
|
9
|
+
DEFAULT_MAX_BODY_SIZE = 100 * 1024 * 1024
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def body_size_limit(value=DEFAULT_MAX_BODY_SIZE):
|
|
13
|
+
"""Accept a positive integer byte limit; there is no unlimited sentinel."""
|
|
14
|
+
if isinstance(value, bool):
|
|
15
|
+
raise ValueError('max_body_size must be a positive integer in bytes')
|
|
16
|
+
try:
|
|
17
|
+
limit = int(value)
|
|
18
|
+
except (TypeError, ValueError, OverflowError):
|
|
19
|
+
limit = 0
|
|
20
|
+
if limit <= 0 or str(limit) != str(value).strip():
|
|
21
|
+
raise ValueError('max_body_size must be a positive integer in bytes')
|
|
22
|
+
return limit
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class _BodyStream:
|
|
26
|
+
"""A binary view beginning at the input's original position.
|
|
27
|
+
|
|
28
|
+
Closing an owned spool releases its temporary file. Caller-owned files
|
|
29
|
+
remain open. The view supplies Requests with its remaining byte length.
|
|
30
|
+
"""
|
|
31
|
+
def __init__(self, source, start, size, owned=False):
|
|
32
|
+
self.source = source
|
|
33
|
+
self.start = start
|
|
34
|
+
self.len = size
|
|
35
|
+
self.owned = owned
|
|
36
|
+
self.closed = False
|
|
37
|
+
|
|
38
|
+
def read(self, size=-1):
|
|
39
|
+
if self.closed:
|
|
40
|
+
raise ValueError('upload body is closed')
|
|
41
|
+
remaining = max(0, self.len - self.tell())
|
|
42
|
+
amount = remaining if size < 0 else min(size, remaining)
|
|
43
|
+
return self.source.read(amount)
|
|
44
|
+
|
|
45
|
+
def tell(self):
|
|
46
|
+
return self.source.tell() - self.start
|
|
47
|
+
|
|
48
|
+
def seek(self, offset, whence=0):
|
|
49
|
+
if self.closed:
|
|
50
|
+
raise ValueError('upload body is closed')
|
|
51
|
+
if whence == 1:
|
|
52
|
+
offset += self.tell()
|
|
53
|
+
elif whence == 2:
|
|
54
|
+
offset += self.len
|
|
55
|
+
elif whence != 0:
|
|
56
|
+
raise ValueError('invalid seek mode')
|
|
57
|
+
if offset < 0:
|
|
58
|
+
raise ValueError('negative upload position')
|
|
59
|
+
self.source.seek(self.start + offset)
|
|
60
|
+
return self.tell()
|
|
61
|
+
|
|
62
|
+
def close(self):
|
|
63
|
+
if self.owned and not self.closed:
|
|
64
|
+
self.source.close()
|
|
65
|
+
self.closed = True
|
|
66
|
+
|
|
67
|
+
def close_after_response(self, response, **kwargs):
|
|
68
|
+
# Redirects may replay the same body. Keep it until the final response.
|
|
69
|
+
if not response.is_redirect:
|
|
70
|
+
self.close()
|
|
71
|
+
return response
|
|
72
|
+
|
|
73
|
+
def __del__(self):
|
|
74
|
+
# Requests has no error/offline-completion hook. Explicit close and the
|
|
75
|
+
# response hook cover normal use; abandoned requests need the fallback.
|
|
76
|
+
self.close()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class ChunkedBody:
|
|
80
|
+
"""Replay a signed payload without exposing a length to Requests.
|
|
81
|
+
|
|
82
|
+
A length here makes Requests add Content-Length after auth even though the
|
|
83
|
+
request already has Transfer-Encoding: chunked.
|
|
84
|
+
"""
|
|
85
|
+
def __init__(self, body):
|
|
86
|
+
self.body = body
|
|
87
|
+
|
|
88
|
+
def __iter__(self):
|
|
89
|
+
if isinstance(self.body, _BodyStream):
|
|
90
|
+
self.body.seek(0)
|
|
91
|
+
while True:
|
|
92
|
+
chunk = self.body.read(CHUNK_SIZE)
|
|
93
|
+
if not chunk:
|
|
94
|
+
break
|
|
95
|
+
yield chunk
|
|
96
|
+
elif self.body:
|
|
97
|
+
yield self.body
|
|
98
|
+
|
|
99
|
+
def close(self):
|
|
100
|
+
if isinstance(self.body, _BodyStream):
|
|
101
|
+
self.body.close()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def prepare_body(body, limit):
|
|
105
|
+
"""Hash bounded chunks, rewind binary files, and spool other streams.
|
|
106
|
+
|
|
107
|
+
Return (transmission body, size, base64 MD5, hex SHA256). Read from the
|
|
108
|
+
current position; never close a file owned by the caller. Input sizes are
|
|
109
|
+
measured while reading, independent of Content-Length or iterator hints.
|
|
110
|
+
"""
|
|
111
|
+
md5 = hashlib.md5()
|
|
112
|
+
sha256 = hashlib.sha256()
|
|
113
|
+
total = 0
|
|
114
|
+
spool = None
|
|
115
|
+
start = None
|
|
116
|
+
source = None
|
|
117
|
+
direct = isinstance(body, bytes) or body is None
|
|
118
|
+
try:
|
|
119
|
+
if isinstance(body, memoryview):
|
|
120
|
+
body = body.cast('B')
|
|
121
|
+
if direct:
|
|
122
|
+
chunks = (memoryview(body or b'')[offset:offset + CHUNK_SIZE]
|
|
123
|
+
for offset in range(0, len(body or b''), CHUNK_SIZE))
|
|
124
|
+
if len(body or b'') > limit:
|
|
125
|
+
raise ValueError(
|
|
126
|
+
f'upload exceeds max_body_size ({limit} bytes)')
|
|
127
|
+
elif isinstance(body, (str, bytearray, memoryview)):
|
|
128
|
+
if len(body) > limit:
|
|
129
|
+
raise ValueError(
|
|
130
|
+
f'upload exceeds max_body_size ({limit} bytes)')
|
|
131
|
+
chunks = (body[offset:offset + CHUNK_SIZE]
|
|
132
|
+
for offset in range(0, len(body), CHUNK_SIZE))
|
|
133
|
+
elif hasattr(body, 'read'):
|
|
134
|
+
source = body
|
|
135
|
+
try:
|
|
136
|
+
start = body.tell()
|
|
137
|
+
body.seek(start)
|
|
138
|
+
except (AttributeError, OSError, ValueError):
|
|
139
|
+
start = None
|
|
140
|
+
|
|
141
|
+
def read_chunks():
|
|
142
|
+
while True:
|
|
143
|
+
chunk = body.read(min(CHUNK_SIZE, limit - total + 1))
|
|
144
|
+
if chunk is None:
|
|
145
|
+
raise ValueError('upload read returned None; '
|
|
146
|
+
'nonblocking inputs are unsupported')
|
|
147
|
+
if not chunk:
|
|
148
|
+
break
|
|
149
|
+
yield chunk
|
|
150
|
+
|
|
151
|
+
chunks = read_chunks()
|
|
152
|
+
else:
|
|
153
|
+
chunks = iter(body)
|
|
154
|
+
|
|
155
|
+
for chunk in chunks:
|
|
156
|
+
if not isinstance(chunk, (str, bytes, bytearray, memoryview)):
|
|
157
|
+
raise TypeError('upload chunks must be bytes or text')
|
|
158
|
+
if isinstance(chunk, memoryview):
|
|
159
|
+
chunk = chunk.cast('B')
|
|
160
|
+
# Reject an oversized yielded chunk before encoding/copying it or
|
|
161
|
+
# writing it to disk. UTF-8 needs at least one byte per character.
|
|
162
|
+
if len(chunk) > limit - total:
|
|
163
|
+
raise ValueError(
|
|
164
|
+
f'upload exceeds max_body_size ({limit} bytes)')
|
|
165
|
+
# Iterate over large externally yielded chunks in bounded slices.
|
|
166
|
+
for offset in range(0, len(chunk), CHUNK_SIZE):
|
|
167
|
+
piece = chunk[offset:offset + CHUNK_SIZE]
|
|
168
|
+
if isinstance(piece, str):
|
|
169
|
+
piece = piece.encode('utf-8')
|
|
170
|
+
size = len(piece)
|
|
171
|
+
if size > limit - total:
|
|
172
|
+
raise ValueError(
|
|
173
|
+
f'upload exceeds max_body_size ({limit} bytes)')
|
|
174
|
+
if (isinstance(chunk, str) and source is not None
|
|
175
|
+
and start is not None and spool is None and total):
|
|
176
|
+
raise TypeError('seekable upload changed from bytes '
|
|
177
|
+
'to text while reading')
|
|
178
|
+
if not direct and (start is None or isinstance(chunk, str)
|
|
179
|
+
or spool is not None):
|
|
180
|
+
if spool is None:
|
|
181
|
+
spool = tempfile.SpooledTemporaryFile(
|
|
182
|
+
max_size=SPOOL_MEMORY_SIZE, mode='w+b')
|
|
183
|
+
# Roll over before writing the threshold-crossing chunk.
|
|
184
|
+
if total + size > SPOOL_MEMORY_SIZE:
|
|
185
|
+
spool.rollover()
|
|
186
|
+
spool.write(piece)
|
|
187
|
+
total += size
|
|
188
|
+
md5.update(piece)
|
|
189
|
+
sha256.update(piece)
|
|
190
|
+
|
|
191
|
+
if spool is not None:
|
|
192
|
+
spool.seek(0)
|
|
193
|
+
if total <= SPOOL_MEMORY_SIZE:
|
|
194
|
+
prepared = spool.read()
|
|
195
|
+
spool.close()
|
|
196
|
+
else:
|
|
197
|
+
prepared = _BodyStream(spool, 0, total, owned=True)
|
|
198
|
+
spool = None
|
|
199
|
+
elif source is not None and start is not None and total:
|
|
200
|
+
source.seek(start)
|
|
201
|
+
prepared = _BodyStream(source, start, total)
|
|
202
|
+
elif direct:
|
|
203
|
+
prepared = body
|
|
204
|
+
else:
|
|
205
|
+
# Empty text/iterables need no spool or open temporary resource.
|
|
206
|
+
prepared = b''
|
|
207
|
+
encoded_md5 = base64.b64encode(md5.digest()).decode('ascii')
|
|
208
|
+
return prepared, total, encoded_md5, sha256.hexdigest()
|
|
209
|
+
except BaseException:
|
|
210
|
+
if spool is not None:
|
|
211
|
+
spool.close()
|
|
212
|
+
raise
|
|
213
|
+
finally:
|
|
214
|
+
if source is not None and start is not None:
|
|
215
|
+
source.seek(start)
|