hegemony-steps-http 0.46.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hegemony_steps_http/__init__.py +33 -0
- hegemony_steps_http/docs/http.request.md +198 -0
- hegemony_steps_http/jsonpath.py +121 -0
- hegemony_steps_http/request.py +1452 -0
- hegemony_steps_http/security.py +314 -0
- hegemony_steps_http-0.46.1.dist-info/METADATA +40 -0
- hegemony_steps_http-0.46.1.dist-info/RECORD +10 -0
- hegemony_steps_http-0.46.1.dist-info/WHEEL +4 -0
- hegemony_steps_http-0.46.1.dist-info/entry_points.txt +3 -0
- hegemony_steps_http-0.46.1.dist-info/licenses/LICENSE-AGPL-3.0-or-later.txt +235 -0
|
@@ -0,0 +1,1452 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2025-2026 Rexonix s.r.o.
|
|
2
|
+
# SPDX-FileContributor: Jakub Trávník <jakub.travnik@gmail.com>
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
5
|
+
|
|
6
|
+
"""HttpRequestHandler: send one HTTP(S) request and return the response as step output.
|
|
7
|
+
|
|
8
|
+
The worker sends the request itself, through its own httpx client, so every
|
|
9
|
+
outgoing request (redirect hops and next pages included) walks the flow's
|
|
10
|
+
egress gate (``services.enforce_egress``) and the step's own destination
|
|
11
|
+
check before a connection is opened. Redirects are followed here rather
|
|
12
|
+
than by httpx, so a redirect body is never read and the credentials and
|
|
13
|
+
author-set headers stay with the origin the author wrote.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
import base64
|
|
20
|
+
import codecs
|
|
21
|
+
import hashlib
|
|
22
|
+
import json
|
|
23
|
+
import re
|
|
24
|
+
import ssl
|
|
25
|
+
import time
|
|
26
|
+
import zlib
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from functools import cached_property
|
|
29
|
+
from typing import Any, Literal
|
|
30
|
+
|
|
31
|
+
import httpx
|
|
32
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
33
|
+
|
|
34
|
+
from hegemony_step_sdk import (
|
|
35
|
+
MAX_CHAIN_OUTPUT_CHARS,
|
|
36
|
+
BaseHandler,
|
|
37
|
+
HandlerContext,
|
|
38
|
+
HandlerResult,
|
|
39
|
+
HandlerServices,
|
|
40
|
+
HandlerTargeting,
|
|
41
|
+
InvalidConfigError,
|
|
42
|
+
StepKind,
|
|
43
|
+
)
|
|
44
|
+
from hegemony_step_sdk.trust import add_trusted_cas, org_trusted_ca_pem
|
|
45
|
+
|
|
46
|
+
from .jsonpath import PathError, get_path, parse_path, replace_at
|
|
47
|
+
from .security import (
|
|
48
|
+
MIN_SCRUB_LENGTH,
|
|
49
|
+
SCHEME_PORTS,
|
|
50
|
+
RequestRefused,
|
|
51
|
+
Scrubber,
|
|
52
|
+
check_destination,
|
|
53
|
+
filtered_response_headers,
|
|
54
|
+
is_bare_secret_ref,
|
|
55
|
+
is_templated,
|
|
56
|
+
resolve_secret_text,
|
|
57
|
+
same_origin,
|
|
58
|
+
url_host,
|
|
59
|
+
url_port,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
HANDLER_ID = "http.request"
|
|
63
|
+
|
|
64
|
+
#: Methods that never carry a request body here.
|
|
65
|
+
BODYLESS_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
|
|
66
|
+
|
|
67
|
+
MIB = 1024 * 1024
|
|
68
|
+
#: Content encodings the step decodes itself, within the size budget; it asks
|
|
69
|
+
#: servers for nothing else.
|
|
70
|
+
ACCEPT_ENCODING = "gzip, deflate"
|
|
71
|
+
_GZIP_ENCODINGS = frozenset({"gzip", "x-gzip"})
|
|
72
|
+
_IDENTITY_ENCODINGS = frozenset({"", "identity"})
|
|
73
|
+
#: Raw bytes a compressed body may take beyond the decoded budget (headers,
|
|
74
|
+
#: trailers and incompressible blocks), so a stream that expands to nothing
|
|
75
|
+
#: cannot be read forever.
|
|
76
|
+
_RAW_SLACK_BYTES = MIB
|
|
77
|
+
DEFAULT_MAX_RESPONSE_BYTES = MIB
|
|
78
|
+
MAX_RESPONSE_BYTES_LIMIT = 16 * MIB
|
|
79
|
+
MAX_PAGES_LIMIT = 100
|
|
80
|
+
MAX_TIMEOUT_SECONDS = 600
|
|
81
|
+
#: Redirect hops followed per request before the step gives up.
|
|
82
|
+
MAX_REDIRECTS = 10
|
|
83
|
+
#: Characters of the last response body kept in ``output.body``.
|
|
84
|
+
MAX_BODY_OUTPUT_CHARS = 64 * 1024
|
|
85
|
+
#: Largest ``output.data``, in bytes of compact JSON. Step outputs are chained
|
|
86
|
+
#: into later steps' templates and snapshots, so they stay small: larger data
|
|
87
|
+
#: is left out (``data_truncated``), while Extract still reads all of it.
|
|
88
|
+
MAX_DATA_OUTPUT_BYTES = 64 * 1024
|
|
89
|
+
#: Characters of the last response body kept in the evidence record.
|
|
90
|
+
EVIDENCE_PREVIEW_CHARS = MAX_CHAIN_OUTPUT_CHARS
|
|
91
|
+
|
|
92
|
+
#: A header name (an RFC 9110 token).
|
|
93
|
+
_HEADER_NAME_RE = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$")
|
|
94
|
+
#: Control characters a header value may not hold (tab is allowed).
|
|
95
|
+
_HEADER_VALUE_BAD_RE = re.compile(r"[\x00-\x08\x0a-\x1f\x7f]")
|
|
96
|
+
#: Headers the step sets itself (framing, routing): never from config.
|
|
97
|
+
_RESERVED_HEADERS = frozenset({"host", "content-length", "transfer-encoding", "connection"})
|
|
98
|
+
#: Author-set headers a redirect to another origin keeps. Every other header
|
|
99
|
+
#: the author set, and the credentials, stay with the original origin.
|
|
100
|
+
_CROSS_ORIGIN_SAFE_HEADERS = frozenset({"accept", "accept-language", "content-type", "user-agent"})
|
|
101
|
+
#: Names of extracted values (``steps.<id>.output.extracted.<name>``).
|
|
102
|
+
_EXTRACT_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$")
|
|
103
|
+
_PEM_CERTIFICATE = "-----BEGIN CERTIFICATE-----"
|
|
104
|
+
_LITERAL_SCHEME_RE = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*)://")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def parse_status_spec(spec: str) -> list[tuple[int, int]]:
|
|
108
|
+
"""Parse an expected-status spec like ``200-299,304`` into ranges.
|
|
109
|
+
|
|
110
|
+
Raises ``ValueError`` for malformed entries.
|
|
111
|
+
"""
|
|
112
|
+
ranges: list[tuple[int, int]] = []
|
|
113
|
+
for raw in spec.split(","):
|
|
114
|
+
entry = raw.strip()
|
|
115
|
+
if not entry:
|
|
116
|
+
continue
|
|
117
|
+
try:
|
|
118
|
+
if "-" in entry:
|
|
119
|
+
low_raw, high_raw = entry.split("-", 1)
|
|
120
|
+
low, high = int(low_raw), int(high_raw)
|
|
121
|
+
else:
|
|
122
|
+
low = high = int(entry)
|
|
123
|
+
except ValueError:
|
|
124
|
+
raise ValueError(f"invalid status entry: {entry}") from None
|
|
125
|
+
if not (100 <= low <= 599 and 100 <= high <= 599 and low <= high):
|
|
126
|
+
raise ValueError(f"invalid status entry: {entry}")
|
|
127
|
+
ranges.append((low, high))
|
|
128
|
+
if not ranges:
|
|
129
|
+
raise ValueError("expected_status must list at least one status or range")
|
|
130
|
+
return ranges
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def status_matches(status: int, ranges: list[tuple[int, int]]) -> bool:
|
|
134
|
+
"""Whether ``status`` falls in one of ``ranges``."""
|
|
135
|
+
return any(low <= status <= high for low, high in ranges)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class HttpRequestConfig(BaseModel):
|
|
139
|
+
"""Config for ``http.request``."""
|
|
140
|
+
|
|
141
|
+
# A number written for a text setting (expected_status: 201, a query or
|
|
142
|
+
# header value) is its text.
|
|
143
|
+
model_config = ConfigDict(extra="allow", coerce_numbers_to_str=True)
|
|
144
|
+
|
|
145
|
+
url: str = Field(
|
|
146
|
+
min_length=1,
|
|
147
|
+
title="URL",
|
|
148
|
+
description=(
|
|
149
|
+
"Address the request goes to, starting with https:// or http://, such as "
|
|
150
|
+
"https://api.example.com/v1/items. May use templates such as "
|
|
151
|
+
"{{ inputs.base_url }}/items. Credentials never go in the URL: use "
|
|
152
|
+
"Authentication."
|
|
153
|
+
),
|
|
154
|
+
json_schema_extra={"x_placeholder": "https://api.example.com/v1/items", "x_col_span": 2},
|
|
155
|
+
)
|
|
156
|
+
method: Literal["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"] = Field(
|
|
157
|
+
default="GET",
|
|
158
|
+
title="Method",
|
|
159
|
+
description=(
|
|
160
|
+
"HTTP method of the request. GET, HEAD and OPTIONS send no body. Defaults to GET."
|
|
161
|
+
),
|
|
162
|
+
)
|
|
163
|
+
query: dict[str, str] = Field(
|
|
164
|
+
default_factory=dict,
|
|
165
|
+
title="Query Parameters",
|
|
166
|
+
description=(
|
|
167
|
+
"Query parameters added to the URL, NAME=value, one per line. A parameter "
|
|
168
|
+
"the URL already carries is replaced. Values may use templates."
|
|
169
|
+
),
|
|
170
|
+
json_schema_extra={
|
|
171
|
+
"x_widget": "env-vars",
|
|
172
|
+
"x_placeholder": "limit=100\nstatus=active",
|
|
173
|
+
"x_rows": 2,
|
|
174
|
+
"x_col_span": 2,
|
|
175
|
+
},
|
|
176
|
+
)
|
|
177
|
+
headers: dict[str, str] = Field(
|
|
178
|
+
default_factory=dict,
|
|
179
|
+
title="Headers",
|
|
180
|
+
description=(
|
|
181
|
+
"Request headers, Name=value, one per line, such as Accept=application/json. "
|
|
182
|
+
"Values may use templates. Put credentials in Authentication instead: a "
|
|
183
|
+
"redirect to another origin drops the credentials and every header set here "
|
|
184
|
+
"except Accept, Accept-Language, Content-Type and User-Agent. Host, "
|
|
185
|
+
"Content-Length, Transfer-Encoding and Connection are set by the step."
|
|
186
|
+
),
|
|
187
|
+
json_schema_extra={
|
|
188
|
+
"x_widget": "env-vars",
|
|
189
|
+
"x_placeholder": "Accept=application/json",
|
|
190
|
+
"x_rows": 2,
|
|
191
|
+
"x_col_span": 2,
|
|
192
|
+
},
|
|
193
|
+
)
|
|
194
|
+
json_body: Any = Field(
|
|
195
|
+
default=None,
|
|
196
|
+
title="JSON Body",
|
|
197
|
+
description=(
|
|
198
|
+
"Request body sent as JSON, with Content-Type application/json unless "
|
|
199
|
+
"Headers set another. Text is parsed as JSON, so a template that renders "
|
|
200
|
+
'JSON works, such as {"name": "{{ inputs.name }}"}. Not allowed with '
|
|
201
|
+
"GET, HEAD or OPTIONS, or together with Text Body."
|
|
202
|
+
),
|
|
203
|
+
json_schema_extra={
|
|
204
|
+
"x_placeholder": '{"name": "{{ inputs.name }}"}',
|
|
205
|
+
"x_rows": 6,
|
|
206
|
+
"x_col_span": 2,
|
|
207
|
+
},
|
|
208
|
+
)
|
|
209
|
+
text_body: str = Field(
|
|
210
|
+
default="",
|
|
211
|
+
title="Text Body",
|
|
212
|
+
description=(
|
|
213
|
+
"Request body sent as written, with Content-Type text/plain; charset=utf-8 "
|
|
214
|
+
"unless Headers set another. Not allowed with GET, HEAD or OPTIONS, or "
|
|
215
|
+
"together with JSON Body."
|
|
216
|
+
),
|
|
217
|
+
json_schema_extra={"x_rows": 4, "x_col_span": 2, "x_advanced": True},
|
|
218
|
+
)
|
|
219
|
+
auth_type: Literal["none", "bearer", "basic", "header"] = Field(
|
|
220
|
+
default="none",
|
|
221
|
+
title="Authentication",
|
|
222
|
+
description=(
|
|
223
|
+
"How the request authenticates: none; a bearer token (Authorization: Bearer "
|
|
224
|
+
"and the secret); basic, with a username and the secret as password; or a "
|
|
225
|
+
"custom header carrying the secret. Credentials go only to the URL's own "
|
|
226
|
+
"scheme, host and port: a redirect elsewhere drops them. They need an "
|
|
227
|
+
"https:// URL unless Allow credentials over HTTP is on."
|
|
228
|
+
),
|
|
229
|
+
json_schema_extra={
|
|
230
|
+
"x_option_labels": {
|
|
231
|
+
"none": "None",
|
|
232
|
+
"bearer": "Bearer token",
|
|
233
|
+
"basic": "Basic (username and password)",
|
|
234
|
+
"header": "Custom header",
|
|
235
|
+
}
|
|
236
|
+
},
|
|
237
|
+
)
|
|
238
|
+
auth_secret: str = Field(
|
|
239
|
+
default="",
|
|
240
|
+
title="Secret",
|
|
241
|
+
description=(
|
|
242
|
+
"The token or password: a secret reference such as "
|
|
243
|
+
"vault://orgs/acme/secrets/api/token, a {{ secret('...') }} template, or "
|
|
244
|
+
"literal text. A value resolved from a reference is redacted from the run's "
|
|
245
|
+
"logs; literal text is not, so prefer a reference."
|
|
246
|
+
),
|
|
247
|
+
json_schema_extra={
|
|
248
|
+
"x_placeholder": "vault://orgs/acme/secrets/api/token",
|
|
249
|
+
"x_hide_when": {"field": "auth_type", "value": "none"},
|
|
250
|
+
"x_col_span": 2,
|
|
251
|
+
},
|
|
252
|
+
)
|
|
253
|
+
auth_username: str = Field(
|
|
254
|
+
default="",
|
|
255
|
+
title="Username",
|
|
256
|
+
description=(
|
|
257
|
+
"Username for basic authentication; it may not contain a colon. May use templates."
|
|
258
|
+
),
|
|
259
|
+
json_schema_extra={"x_show_when": {"field": "auth_type", "value": "basic"}},
|
|
260
|
+
)
|
|
261
|
+
auth_header_name: str = Field(
|
|
262
|
+
default="Authorization",
|
|
263
|
+
title="Header Name",
|
|
264
|
+
description=(
|
|
265
|
+
"Header that carries the secret when Authentication is Custom header, such "
|
|
266
|
+
"as X-API-Key. Defaults to Authorization."
|
|
267
|
+
),
|
|
268
|
+
json_schema_extra={
|
|
269
|
+
"x_placeholder": "X-API-Key",
|
|
270
|
+
"x_show_when": {"field": "auth_type", "value": "header"},
|
|
271
|
+
},
|
|
272
|
+
)
|
|
273
|
+
auth_prefix: str = Field(
|
|
274
|
+
default="",
|
|
275
|
+
title="Value Prefix",
|
|
276
|
+
description=(
|
|
277
|
+
"Text sent before the secret in the custom header, such as 'Token ' with its "
|
|
278
|
+
"trailing space. Empty sends the secret alone."
|
|
279
|
+
),
|
|
280
|
+
json_schema_extra={
|
|
281
|
+
"x_placeholder": "Token ",
|
|
282
|
+
"x_show_when": {"field": "auth_type", "value": "header"},
|
|
283
|
+
},
|
|
284
|
+
)
|
|
285
|
+
allow_insecure_auth: bool = Field(
|
|
286
|
+
default=False,
|
|
287
|
+
title="Allow credentials over HTTP",
|
|
288
|
+
description=(
|
|
289
|
+
"Send the Authentication credential to a plain http:// URL. Off by default, "
|
|
290
|
+
"because over HTTP the token or password travels unencrypted and anyone on "
|
|
291
|
+
"the network path can read it; turn on only for an internal endpoint that "
|
|
292
|
+
"offers no HTTPS."
|
|
293
|
+
),
|
|
294
|
+
json_schema_extra={"x_advanced": True},
|
|
295
|
+
)
|
|
296
|
+
expected_status: str = Field(
|
|
297
|
+
default="200-299",
|
|
298
|
+
title="Expected Status",
|
|
299
|
+
description=(
|
|
300
|
+
"Comma-separated status codes and ranges that count as success, such as "
|
|
301
|
+
"200-299,304. Defaults to 200-299. With pagination every page must match."
|
|
302
|
+
),
|
|
303
|
+
json_schema_extra={"x_placeholder": "200-299"},
|
|
304
|
+
)
|
|
305
|
+
request_timeout_seconds: int = Field(
|
|
306
|
+
default=30,
|
|
307
|
+
ge=1,
|
|
308
|
+
le=MAX_TIMEOUT_SECONDS,
|
|
309
|
+
title="Timeout (sec)",
|
|
310
|
+
description=(
|
|
311
|
+
"Time limit for each request, in seconds: connecting, sending and reading the "
|
|
312
|
+
"whole response. Redirect hops and pages each get their own. Defaults to 30; "
|
|
313
|
+
"1 to 600."
|
|
314
|
+
),
|
|
315
|
+
)
|
|
316
|
+
follow_redirects: bool = Field(
|
|
317
|
+
default=False,
|
|
318
|
+
title="Follow redirects",
|
|
319
|
+
description=(
|
|
320
|
+
"Follow 3xx redirects, at most 10, and report the final response. Off by "
|
|
321
|
+
"default, when the redirect response itself is the result. Every hop is "
|
|
322
|
+
"checked against the egress policy, and a redirect to a scheme other than "
|
|
323
|
+
"http or https, or one that would send the request body to another origin, "
|
|
324
|
+
"is refused."
|
|
325
|
+
),
|
|
326
|
+
)
|
|
327
|
+
verify_tls: bool = Field(
|
|
328
|
+
default=True,
|
|
329
|
+
title="Verify TLS certificate",
|
|
330
|
+
description=(
|
|
331
|
+
"Validate the server's certificate on HTTPS requests. Turn off only for test "
|
|
332
|
+
"endpoints with self-signed certificates, and prefer CA Bundle. On by default."
|
|
333
|
+
),
|
|
334
|
+
json_schema_extra={"x_advanced": True},
|
|
335
|
+
)
|
|
336
|
+
ca_bundle: str = Field(
|
|
337
|
+
default="",
|
|
338
|
+
title="CA Bundle",
|
|
339
|
+
description=(
|
|
340
|
+
"PEM certificates to trust for this request instead of the public certificate "
|
|
341
|
+
"authorities and the organization's trusted CAs, which pins them: the PEM text, "
|
|
342
|
+
"a secret reference, or a {{ secret('...') }} template. Empty trusts the public "
|
|
343
|
+
"authorities plus the organization's trusted CAs (Settings, Trusted CAs). "
|
|
344
|
+
"Ignored when Verify TLS certificate is off."
|
|
345
|
+
),
|
|
346
|
+
json_schema_extra={
|
|
347
|
+
"x_placeholder": "-----BEGIN CERTIFICATE-----",
|
|
348
|
+
"x_show_when": {"field": "verify_tls", "value": True},
|
|
349
|
+
"x_rows": 4,
|
|
350
|
+
"x_col_span": 2,
|
|
351
|
+
"x_advanced": True,
|
|
352
|
+
},
|
|
353
|
+
)
|
|
354
|
+
max_response_bytes: int = Field(
|
|
355
|
+
default=DEFAULT_MAX_RESPONSE_BYTES,
|
|
356
|
+
ge=1,
|
|
357
|
+
le=MAX_RESPONSE_BYTES_LIMIT,
|
|
358
|
+
title="Max Response Size (bytes)",
|
|
359
|
+
description=(
|
|
360
|
+
"Largest response body the step reads, in bytes, summed over all pages. A "
|
|
361
|
+
"larger response fails the step instead of being cut short. Defaults to "
|
|
362
|
+
"1048576 (1 MiB); at most 16777216 (16 MiB)."
|
|
363
|
+
),
|
|
364
|
+
json_schema_extra={"x_advanced": True},
|
|
365
|
+
)
|
|
366
|
+
extract: dict[str, str] = Field(
|
|
367
|
+
default_factory=dict,
|
|
368
|
+
title="Extract",
|
|
369
|
+
description=(
|
|
370
|
+
"Values to pick from the JSON response, NAME=path one per line, such as "
|
|
371
|
+
"id=results[0].id. A path is keys separated by dots, [n] for list items "
|
|
372
|
+
"(negative counts from the end), and an optional leading $. A missing value "
|
|
373
|
+
"is null. Later steps read them as steps.<id>.output.extracted.NAME."
|
|
374
|
+
),
|
|
375
|
+
json_schema_extra={
|
|
376
|
+
"x_widget": "env-vars",
|
|
377
|
+
"x_placeholder": "id=results[0].id",
|
|
378
|
+
"x_rows": 2,
|
|
379
|
+
"x_col_span": 2,
|
|
380
|
+
},
|
|
381
|
+
)
|
|
382
|
+
sensitive: bool = Field(
|
|
383
|
+
default=False,
|
|
384
|
+
title="Sensitive response",
|
|
385
|
+
description=(
|
|
386
|
+
"Treat the response as secret: its body, data and extracted values stay out "
|
|
387
|
+
"of the output and evidence, which keep the status, headers, size and SHA-256 "
|
|
388
|
+
"of the body. Off by default."
|
|
389
|
+
),
|
|
390
|
+
json_schema_extra={"x_advanced": True},
|
|
391
|
+
)
|
|
392
|
+
paginate_next: str = Field(
|
|
393
|
+
default="",
|
|
394
|
+
title="Next Page Path",
|
|
395
|
+
description=(
|
|
396
|
+
"Path of the next page's URL in each JSON response, such as next. A missing, "
|
|
397
|
+
"null or empty value ends pagination. Next pages are fetched with GET, the "
|
|
398
|
+
"same headers and credentials, and must stay on the first request's scheme, "
|
|
399
|
+
"host and port. Needs GET, Items Path, and Max Pages above 1."
|
|
400
|
+
),
|
|
401
|
+
json_schema_extra={"x_placeholder": "next", "x_advanced": True},
|
|
402
|
+
)
|
|
403
|
+
paginate_items: str = Field(
|
|
404
|
+
default="",
|
|
405
|
+
title="Items Path",
|
|
406
|
+
description=(
|
|
407
|
+
"Path of the item list in each JSON response, such as results. The output's "
|
|
408
|
+
"data is the first page with this list replaced by the items of every page."
|
|
409
|
+
),
|
|
410
|
+
json_schema_extra={"x_placeholder": "results", "x_advanced": True},
|
|
411
|
+
)
|
|
412
|
+
max_pages: int = Field(
|
|
413
|
+
default=1,
|
|
414
|
+
ge=1,
|
|
415
|
+
le=MAX_PAGES_LIMIT,
|
|
416
|
+
title="Max Pages",
|
|
417
|
+
description=(
|
|
418
|
+
"Most pages fetched, the first included. 1, the default, fetches a single "
|
|
419
|
+
"response; at most 100."
|
|
420
|
+
),
|
|
421
|
+
json_schema_extra={"x_advanced": True},
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _url_error(url: str, *, allow_templates: bool) -> str | None:
|
|
426
|
+
"""Why ``url`` cannot be requested, or None."""
|
|
427
|
+
text = url.strip()
|
|
428
|
+
if not text:
|
|
429
|
+
return "'url' is required"
|
|
430
|
+
if allow_templates and is_templated(text):
|
|
431
|
+
literal = _LITERAL_SCHEME_RE.match(text)
|
|
432
|
+
if literal and literal.group(1).lower() not in SCHEME_PORTS:
|
|
433
|
+
return "'url' must start with http:// or https://"
|
|
434
|
+
return None
|
|
435
|
+
try:
|
|
436
|
+
parsed = httpx.URL(text)
|
|
437
|
+
except (httpx.InvalidURL, TypeError, ValueError) as exc:
|
|
438
|
+
return f"'url' is not a valid URL: {exc}"
|
|
439
|
+
if parsed.scheme not in SCHEME_PORTS:
|
|
440
|
+
return "'url' must start with http:// or https://"
|
|
441
|
+
if not parsed.host:
|
|
442
|
+
return "'url' has no host"
|
|
443
|
+
if parsed.userinfo:
|
|
444
|
+
return "'url' must not carry credentials (user:password@); use Authentication instead"
|
|
445
|
+
return None
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _header_value_error(field_name: str, value: str) -> str | None:
|
|
449
|
+
"""Why ``value`` cannot be sent as a header value, or None. Never echoes the value."""
|
|
450
|
+
if _HEADER_VALUE_BAD_RE.search(value):
|
|
451
|
+
return f"{field_name} contains line breaks or control characters"
|
|
452
|
+
if not value.isascii():
|
|
453
|
+
return f"{field_name} contains non-ASCII characters"
|
|
454
|
+
return None
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _has_json_body(config: HttpRequestConfig) -> bool:
|
|
458
|
+
"""Whether JSON Body is set (blank text counts as unset)."""
|
|
459
|
+
body = config.json_body
|
|
460
|
+
return body is not None and not (isinstance(body, str) and not body.strip())
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _json_body(config: HttpRequestConfig) -> tuple[bool, Any]:
|
|
464
|
+
"""``(present, value)`` of the JSON body; text is parsed. Raises ``ValueError``."""
|
|
465
|
+
if not _has_json_body(config):
|
|
466
|
+
return False, None
|
|
467
|
+
body = config.json_body
|
|
468
|
+
if isinstance(body, str):
|
|
469
|
+
try:
|
|
470
|
+
body = json.loads(body)
|
|
471
|
+
except (ValueError, RecursionError) as exc:
|
|
472
|
+
raise ValueError(f"'json_body' is not valid JSON: {exc}") from None
|
|
473
|
+
try:
|
|
474
|
+
json.dumps(body, allow_nan=False)
|
|
475
|
+
except (TypeError, ValueError) as exc:
|
|
476
|
+
raise ValueError(f"'json_body' cannot be sent as JSON: {exc}") from None
|
|
477
|
+
return True, body
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _paths_errors(config: HttpRequestConfig) -> list[str]:
|
|
481
|
+
"""Errors in the Extract paths and the pagination paths (empty when valid)."""
|
|
482
|
+
errors: list[str] = []
|
|
483
|
+
for name, path in config.extract.items():
|
|
484
|
+
if not _EXTRACT_NAME_RE.match(name):
|
|
485
|
+
errors.append(
|
|
486
|
+
f"'extract' name {name!r} must start with a letter or underscore and use only "
|
|
487
|
+
"letters, digits and underscores (max 64)"
|
|
488
|
+
)
|
|
489
|
+
try:
|
|
490
|
+
parse_path(path)
|
|
491
|
+
except PathError as exc:
|
|
492
|
+
errors.append(f"'extract.{name}': {exc}")
|
|
493
|
+
for field_name in ("paginate_next", "paginate_items"):
|
|
494
|
+
value = getattr(config, field_name).strip()
|
|
495
|
+
if value:
|
|
496
|
+
try:
|
|
497
|
+
parse_path(value)
|
|
498
|
+
except PathError as exc:
|
|
499
|
+
errors.append(f"'{field_name}': {exc}")
|
|
500
|
+
return errors
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def config_errors(config: HttpRequestConfig, *, allow_templates: bool) -> list[str]:
|
|
504
|
+
"""Semantic errors a schema cannot express (empty when valid).
|
|
505
|
+
|
|
506
|
+
With ``allow_templates`` (saving a flow), values still holding template
|
|
507
|
+
syntax pass; the step checks them again once the host has rendered them.
|
|
508
|
+
Messages name fields, never the values of headers or secrets.
|
|
509
|
+
"""
|
|
510
|
+
errors: list[str] = []
|
|
511
|
+
|
|
512
|
+
def unchecked(value: Any) -> bool:
|
|
513
|
+
"""Whether ``value`` is left for the check after rendering."""
|
|
514
|
+
return allow_templates and is_templated(value)
|
|
515
|
+
|
|
516
|
+
error = _url_error(config.url, allow_templates=allow_templates)
|
|
517
|
+
if error:
|
|
518
|
+
errors.append(error)
|
|
519
|
+
|
|
520
|
+
if any(not name.strip() for name in config.query):
|
|
521
|
+
errors.append("'query' names must not be empty")
|
|
522
|
+
|
|
523
|
+
credential_header = _credential_header_name(config)
|
|
524
|
+
for name, value in config.headers.items():
|
|
525
|
+
if not _HEADER_NAME_RE.match(name):
|
|
526
|
+
errors.append(f"'headers' name {name!r} is not a valid header name")
|
|
527
|
+
continue
|
|
528
|
+
lowered = name.lower()
|
|
529
|
+
if lowered in _RESERVED_HEADERS:
|
|
530
|
+
errors.append(f"'headers' must not set {name}: the step sets it")
|
|
531
|
+
elif credential_header and lowered == credential_header.lower():
|
|
532
|
+
errors.append(
|
|
533
|
+
f"'headers' must not set {name}: Authentication sets it; remove it from Headers"
|
|
534
|
+
)
|
|
535
|
+
if not unchecked(value):
|
|
536
|
+
error = _header_value_error(f"'headers.{name}'", value)
|
|
537
|
+
if error:
|
|
538
|
+
errors.append(error)
|
|
539
|
+
|
|
540
|
+
has_json = _has_json_body(config)
|
|
541
|
+
if has_json and not unchecked(config.json_body):
|
|
542
|
+
try:
|
|
543
|
+
_json_body(config)
|
|
544
|
+
except ValueError as exc:
|
|
545
|
+
errors.append(str(exc))
|
|
546
|
+
has_text = bool(config.text_body)
|
|
547
|
+
if has_json and has_text:
|
|
548
|
+
errors.append("'json_body' and 'text_body' cannot both be set")
|
|
549
|
+
if (has_json or has_text) and config.method in BODYLESS_METHODS:
|
|
550
|
+
errors.append(f"a {config.method} request sends no body: clear JSON Body and Text Body")
|
|
551
|
+
|
|
552
|
+
if config.auth_type != "none" and not config.auth_secret.strip():
|
|
553
|
+
errors.append(f"'auth_secret' is required for {config.auth_type} authentication")
|
|
554
|
+
if (
|
|
555
|
+
config.auth_type != "none"
|
|
556
|
+
and not config.allow_insecure_auth
|
|
557
|
+
and not unchecked(config.url)
|
|
558
|
+
and config.url.strip().lower().startswith("http://")
|
|
559
|
+
):
|
|
560
|
+
errors.append(
|
|
561
|
+
"Authentication needs an https:// URL: over http:// the credential travels "
|
|
562
|
+
"unencrypted (turn on 'allow_insecure_auth' for an internal endpoint without HTTPS)"
|
|
563
|
+
)
|
|
564
|
+
if config.auth_type == "basic":
|
|
565
|
+
username = config.auth_username
|
|
566
|
+
if not username.strip():
|
|
567
|
+
errors.append("'auth_username' is required for basic authentication")
|
|
568
|
+
elif not unchecked(username):
|
|
569
|
+
# Base64-encoded into the header, so any text but a colon or a
|
|
570
|
+
# control character will do.
|
|
571
|
+
if ":" in username:
|
|
572
|
+
errors.append("'auth_username' must not contain a colon")
|
|
573
|
+
if _HEADER_VALUE_BAD_RE.search(username):
|
|
574
|
+
errors.append("'auth_username' contains line breaks or control characters")
|
|
575
|
+
if config.auth_type == "header":
|
|
576
|
+
name = config.auth_header_name.strip()
|
|
577
|
+
if not _HEADER_NAME_RE.match(name):
|
|
578
|
+
errors.append("'auth_header_name' is not a valid header name")
|
|
579
|
+
elif name.lower() in _RESERVED_HEADERS:
|
|
580
|
+
errors.append(f"'auth_header_name' must not be {name}: the step sets it")
|
|
581
|
+
if not unchecked(config.auth_prefix):
|
|
582
|
+
error = _header_value_error("'auth_prefix'", config.auth_prefix)
|
|
583
|
+
if error:
|
|
584
|
+
errors.append(error)
|
|
585
|
+
|
|
586
|
+
ca_bundle = config.ca_bundle.strip()
|
|
587
|
+
if (
|
|
588
|
+
config.verify_tls
|
|
589
|
+
and ca_bundle
|
|
590
|
+
and not is_templated(ca_bundle)
|
|
591
|
+
and not is_bare_secret_ref(ca_bundle)
|
|
592
|
+
and _PEM_CERTIFICATE not in ca_bundle
|
|
593
|
+
):
|
|
594
|
+
errors.append(
|
|
595
|
+
"'ca_bundle' must hold PEM certificates (-----BEGIN CERTIFICATE-----), a secret "
|
|
596
|
+
"reference, or a {{ secret('...') }} template"
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
if not unchecked(config.expected_status):
|
|
600
|
+
try:
|
|
601
|
+
parse_status_spec(config.expected_status)
|
|
602
|
+
except ValueError as exc:
|
|
603
|
+
errors.append(f"'expected_status': {exc}")
|
|
604
|
+
|
|
605
|
+
errors.extend(_paths_errors(config))
|
|
606
|
+
if config.sensitive and config.extract:
|
|
607
|
+
errors.append(
|
|
608
|
+
"'extract' has no effect on a sensitive response, whose values stay out of the "
|
|
609
|
+
"output: clear Extract or turn off Sensitive response"
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
paginate_next = config.paginate_next.strip()
|
|
613
|
+
paginate_items = config.paginate_items.strip()
|
|
614
|
+
if paginate_next or paginate_items or config.max_pages > 1:
|
|
615
|
+
if not paginate_next:
|
|
616
|
+
errors.append("pagination needs 'paginate_next', the path of the next page's URL")
|
|
617
|
+
if not paginate_items:
|
|
618
|
+
errors.append("pagination needs 'paginate_items', the path of each page's items")
|
|
619
|
+
if config.max_pages < 2:
|
|
620
|
+
errors.append("pagination needs 'max_pages' above 1 (1 fetches only the first page)")
|
|
621
|
+
if config.method != "GET":
|
|
622
|
+
errors.append("pagination needs the GET method")
|
|
623
|
+
return errors
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def _credential_header_name(config: HttpRequestConfig) -> str | None:
|
|
627
|
+
"""The request header Authentication sets, or None."""
|
|
628
|
+
if config.auth_type in {"bearer", "basic"}:
|
|
629
|
+
return "Authorization"
|
|
630
|
+
if config.auth_type == "header":
|
|
631
|
+
return config.auth_header_name.strip() or "Authorization"
|
|
632
|
+
return None
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
@dataclass(frozen=True)
|
|
636
|
+
class _Credentials:
|
|
637
|
+
"""The request header Authentication sets, and the values to scrub from reports."""
|
|
638
|
+
|
|
639
|
+
header: str | None = None
|
|
640
|
+
value: str = field(default="", repr=False)
|
|
641
|
+
secrets: tuple[str, ...] = field(default=(), repr=False)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
@dataclass(frozen=True)
|
|
645
|
+
class HttpResponse:
|
|
646
|
+
"""What :meth:`HttpRequestHandler.send` produced.
|
|
647
|
+
|
|
648
|
+
``result`` is what an ``http.request`` step reports. ``data`` is the whole
|
|
649
|
+
parsed JSON body of the final response (the pages combined, when
|
|
650
|
+
paginating), with known credentials scrubbed and without the size cap
|
|
651
|
+
``output.data`` has; None when the body is not JSON, the response is
|
|
652
|
+
sensitive, or no response came. ``transient`` is true when no usable
|
|
653
|
+
response came because of a timeout or a dropped or refused connection,
|
|
654
|
+
so the request may not have reached the server and a read may be tried
|
|
655
|
+
again.
|
|
656
|
+
"""
|
|
657
|
+
|
|
658
|
+
result: HandlerResult
|
|
659
|
+
data: Any = None
|
|
660
|
+
transient: bool = False
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
@dataclass
|
|
664
|
+
class _Exchange:
|
|
665
|
+
"""One request the step sent (a redirect hop or a page) and the status it got."""
|
|
666
|
+
|
|
667
|
+
method: str
|
|
668
|
+
url: str
|
|
669
|
+
status: int | None = None
|
|
670
|
+
headers: dict[str, str] = field(default_factory=dict)
|
|
671
|
+
elapsed_ms: int = 0
|
|
672
|
+
|
|
673
|
+
def record(self) -> dict[str, Any]:
|
|
674
|
+
"""The exchange as recorded in the evidence."""
|
|
675
|
+
return {
|
|
676
|
+
"method": self.method,
|
|
677
|
+
"url": self.url,
|
|
678
|
+
"status": self.status,
|
|
679
|
+
"elapsed_ms": self.elapsed_ms,
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
@dataclass
|
|
684
|
+
class _Page:
|
|
685
|
+
"""The final response of one page, its body read."""
|
|
686
|
+
|
|
687
|
+
exchange: _Exchange
|
|
688
|
+
url: httpx.URL
|
|
689
|
+
body: bytes
|
|
690
|
+
charset: str | None
|
|
691
|
+
|
|
692
|
+
@cached_property
|
|
693
|
+
def text(self) -> str:
|
|
694
|
+
"""The body decoded by its charset (UTF-8 by default), bad bytes replaced.
|
|
695
|
+
|
|
696
|
+
A UTF-8 body drops a leading byte order mark, which JSON parsing refuses.
|
|
697
|
+
"""
|
|
698
|
+
charset = self.charset or "utf-8"
|
|
699
|
+
try:
|
|
700
|
+
if codecs.lookup(charset).name == "utf-8":
|
|
701
|
+
charset = "utf-8-sig"
|
|
702
|
+
return self.body.decode(charset, errors="replace")
|
|
703
|
+
except LookupError:
|
|
704
|
+
return self.body.decode("utf-8-sig", errors="replace")
|
|
705
|
+
|
|
706
|
+
@cached_property
|
|
707
|
+
def json(self) -> tuple[bool, Any]:
|
|
708
|
+
"""``(parsed, value)``: the body as JSON, when it is JSON."""
|
|
709
|
+
if not self.body.strip():
|
|
710
|
+
return False, None
|
|
711
|
+
try:
|
|
712
|
+
return True, json.loads(self.text)
|
|
713
|
+
except (ValueError, RecursionError):
|
|
714
|
+
return False, None
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
class _BodyError(Exception):
|
|
718
|
+
"""The response body cannot be used: too large, or not decodable."""
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
class _ResponseTooLarge(_BodyError):
|
|
722
|
+
"""The response body exceeds the step's byte budget."""
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def _elapsed_ms(started: float) -> int:
|
|
726
|
+
"""Milliseconds since ``started`` (a ``time.perf_counter()`` value), rounded."""
|
|
727
|
+
return round((time.perf_counter() - started) * 1000)
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
class _BoundedDecoder:
|
|
731
|
+
"""Decompress a gzip or deflate body without ever producing more than the budget.
|
|
732
|
+
|
|
733
|
+
The step reads the raw body and decodes it here rather than letting httpx
|
|
734
|
+
decode it, because httpx inflates each chunk in full before the size
|
|
735
|
+
check could see it: a small compressed body could expand to gigabytes.
|
|
736
|
+
"""
|
|
737
|
+
|
|
738
|
+
def __init__(self, encoding: str, too_large: str) -> None:
|
|
739
|
+
"""Start decoding ``encoding``; ``too_large`` is the error once the budget is passed."""
|
|
740
|
+
self._encoding = encoding
|
|
741
|
+
self._too_large = too_large
|
|
742
|
+
self._gzip = encoding in _GZIP_ENCODINGS
|
|
743
|
+
self._raw_deflate = False
|
|
744
|
+
self._started = False
|
|
745
|
+
self._decompressor = zlib.decompressobj(
|
|
746
|
+
zlib.MAX_WBITS | 16 if self._gzip else zlib.MAX_WBITS
|
|
747
|
+
)
|
|
748
|
+
|
|
749
|
+
def decode(self, data: bytes, room: int) -> bytes:
|
|
750
|
+
"""The decompressed form of ``data``; raises once more than ``room`` bytes come out.
|
|
751
|
+
|
|
752
|
+
A gzip body may hold several members, one after another: each is
|
|
753
|
+
decoded within the same budget, and zero bytes padding them are skipped.
|
|
754
|
+
"""
|
|
755
|
+
out = bytearray()
|
|
756
|
+
pending = data
|
|
757
|
+
while pending:
|
|
758
|
+
if self._decompressor.eof:
|
|
759
|
+
if not self._gzip:
|
|
760
|
+
# Deflate data is a single stream; nothing after its end is read.
|
|
761
|
+
break
|
|
762
|
+
pending = pending.lstrip(b"\0")
|
|
763
|
+
if not pending:
|
|
764
|
+
break
|
|
765
|
+
self._decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16)
|
|
766
|
+
try:
|
|
767
|
+
chunk = self._decompressor.decompress(pending, room - len(out) + 1)
|
|
768
|
+
except zlib.error:
|
|
769
|
+
if self._encoding == "deflate" and not self._started and not self._raw_deflate:
|
|
770
|
+
# Some servers send raw deflate without the zlib wrapper.
|
|
771
|
+
self._raw_deflate = True
|
|
772
|
+
self._decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
|
|
773
|
+
continue
|
|
774
|
+
raise _BodyError(f"the response body is not valid {self._encoding} data") from None
|
|
775
|
+
self._started = True
|
|
776
|
+
out.extend(chunk)
|
|
777
|
+
if len(out) > room:
|
|
778
|
+
raise _ResponseTooLarge(self._too_large)
|
|
779
|
+
if self._decompressor.eof:
|
|
780
|
+
pending = self._decompressor.unused_data
|
|
781
|
+
continue
|
|
782
|
+
tail = self._decompressor.unconsumed_tail
|
|
783
|
+
if not chunk and tail == pending:
|
|
784
|
+
break
|
|
785
|
+
pending = tail
|
|
786
|
+
return bytes(out)
|
|
787
|
+
|
|
788
|
+
def finish(self) -> None:
|
|
789
|
+
"""Refuse a body that ended inside its compressed data, cut short on the way."""
|
|
790
|
+
if self._started and not self._decompressor.eof:
|
|
791
|
+
raise _BodyError(
|
|
792
|
+
f"the response body ended before its {self._encoding} data was complete"
|
|
793
|
+
)
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
async def _read_body(response: httpx.Response, remaining: int, limit_text: str) -> bytes:
|
|
797
|
+
"""The response body, decoded, and refused once it passes ``remaining`` bytes.
|
|
798
|
+
|
|
799
|
+
The raw body is read in chunks and decoded here (identity, gzip or
|
|
800
|
+
deflate), so neither the raw nor the decoded body can outgrow the budget.
|
|
801
|
+
A declared Content-Length over the budget is refused before reading,
|
|
802
|
+
unless the response has no body by definition (HEAD, 204, 304) or is
|
|
803
|
+
compressed (the declared length is not the decoded one). A compressed
|
|
804
|
+
body that ends before its compressed data does is refused too.
|
|
805
|
+
"""
|
|
806
|
+
too_large = (
|
|
807
|
+
f"the response body is larger than {limit_text}; raise Max Response Size or request less"
|
|
808
|
+
)
|
|
809
|
+
declared = response.headers.get("content-length", "")
|
|
810
|
+
encoding = response.headers.get("content-encoding", "identity").strip().lower()
|
|
811
|
+
has_body = response.request.method != "HEAD" and response.status_code not in {204, 304}
|
|
812
|
+
identity = encoding in _IDENTITY_ENCODINGS
|
|
813
|
+
if has_body and identity and declared.isdigit() and int(declared) > remaining:
|
|
814
|
+
raise _ResponseTooLarge(
|
|
815
|
+
f"the response body ({declared} bytes) is larger than {limit_text}; raise "
|
|
816
|
+
"Max Response Size or request less"
|
|
817
|
+
)
|
|
818
|
+
if response.is_stream_consumed:
|
|
819
|
+
# Only an in-process transport hands back a body already read (and
|
|
820
|
+
# decoded by httpx); a network response is always still a stream.
|
|
821
|
+
content = response.content
|
|
822
|
+
if len(content) > remaining:
|
|
823
|
+
raise _ResponseTooLarge(too_large)
|
|
824
|
+
return content
|
|
825
|
+
decoder: _BoundedDecoder | None = None
|
|
826
|
+
if not identity and (encoding in _GZIP_ENCODINGS or encoding == "deflate"):
|
|
827
|
+
decoder = _BoundedDecoder(encoding, too_large)
|
|
828
|
+
body = bytearray()
|
|
829
|
+
raw_total = 0
|
|
830
|
+
async for raw in response.aiter_raw():
|
|
831
|
+
if not raw:
|
|
832
|
+
continue
|
|
833
|
+
if not identity and decoder is None:
|
|
834
|
+
raise _BodyError(
|
|
835
|
+
f"the response uses content encoding {encoding!r}, which this step does not "
|
|
836
|
+
f"decode (it asks for {ACCEPT_ENCODING} or none); remove any Accept-Encoding "
|
|
837
|
+
"header that asks for it"
|
|
838
|
+
)
|
|
839
|
+
raw_total += len(raw)
|
|
840
|
+
if decoder is None:
|
|
841
|
+
body.extend(raw)
|
|
842
|
+
else:
|
|
843
|
+
if raw_total > remaining + _RAW_SLACK_BYTES:
|
|
844
|
+
raise _ResponseTooLarge(too_large)
|
|
845
|
+
body.extend(decoder.decode(raw, remaining - len(body)))
|
|
846
|
+
if len(body) > remaining:
|
|
847
|
+
raise _ResponseTooLarge(too_large)
|
|
848
|
+
if decoder is not None:
|
|
849
|
+
decoder.finish()
|
|
850
|
+
return bytes(body)
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
def _strip_for_new_origin(
|
|
854
|
+
headers: httpx.Headers, author_headers: dict[str, str], credential_header: str | None
|
|
855
|
+
) -> None:
|
|
856
|
+
"""Drop credentials and author-set headers before a request leaves the origin."""
|
|
857
|
+
drop = {"authorization", "proxy-authorization"}
|
|
858
|
+
if credential_header:
|
|
859
|
+
drop.add(credential_header.lower())
|
|
860
|
+
drop.update(
|
|
861
|
+
name.lower() for name in author_headers if name.lower() not in _CROSS_ORIGIN_SAFE_HEADERS
|
|
862
|
+
)
|
|
863
|
+
for name in drop:
|
|
864
|
+
headers.pop(name, None)
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
class HttpRequestHandler(BaseHandler):
|
|
868
|
+
"""Send one HTTP(S) request (optionally following next-page links) from the worker."""
|
|
869
|
+
|
|
870
|
+
handler_id = HANDLER_ID
|
|
871
|
+
supported_kinds = [StepKind.ACTION]
|
|
872
|
+
display_name = "HTTP Request"
|
|
873
|
+
description = (
|
|
874
|
+
"Send an HTTP(S) request with any method, headers, body and authentication, and "
|
|
875
|
+
"return the response for later steps."
|
|
876
|
+
)
|
|
877
|
+
category = "Actions"
|
|
878
|
+
targeting = HandlerTargeting(roles=False, ips=False)
|
|
879
|
+
# Secret references reach the handler as written (see resolve_secret_text).
|
|
880
|
+
raw_config_keys = frozenset({"auth_secret", "ca_bundle"})
|
|
881
|
+
config_model = HttpRequestConfig
|
|
882
|
+
default_config = {"method": "GET", "expected_status": "200-299"}
|
|
883
|
+
|
|
884
|
+
def __init__(self, *, transport: httpx.AsyncBaseTransport | None = None) -> None:
|
|
885
|
+
"""Use ``transport`` instead of the network (tests only)."""
|
|
886
|
+
# Tests route the client through a mock transport; the host passes nothing.
|
|
887
|
+
self._transport = transport
|
|
888
|
+
|
|
889
|
+
def validate_config(self, config: dict[str, Any]) -> list[str]:
|
|
890
|
+
"""Validate the step's fields; templated values are checked again once rendered."""
|
|
891
|
+
errors = super().validate_config(config)
|
|
892
|
+
if errors:
|
|
893
|
+
return errors
|
|
894
|
+
return config_errors(HttpRequestConfig.model_validate(config), allow_templates=True)
|
|
895
|
+
|
|
896
|
+
async def execute(self, ctx: HandlerContext) -> HandlerResult:
|
|
897
|
+
"""Validate the rendered config, resolve credentials and TLS trust, then send."""
|
|
898
|
+
try:
|
|
899
|
+
config = self.parse_config(ctx.config, HttpRequestConfig)
|
|
900
|
+
except InvalidConfigError as exc:
|
|
901
|
+
return exc.result()
|
|
902
|
+
return (await self.send(ctx.require_services(), config)).result
|
|
903
|
+
|
|
904
|
+
async def send(self, services: HandlerServices, config: HttpRequestConfig) -> HttpResponse:
|
|
905
|
+
"""Send the request ``config`` describes, as an ``http.request`` step would.
|
|
906
|
+
|
|
907
|
+
For other handlers that speak HTTP (the NetBox steps): every safeguard
|
|
908
|
+
of this step applies, and the result also carries the whole parsed
|
|
909
|
+
response. ``config`` is checked as rendered, so it may not hold
|
|
910
|
+
templates.
|
|
911
|
+
"""
|
|
912
|
+
errors = config_errors(config, allow_templates=False)
|
|
913
|
+
if errors:
|
|
914
|
+
return HttpResponse(InvalidConfigError(HANDLER_ID, errors).result())
|
|
915
|
+
if getattr(services, "enforce_egress", None) is None:
|
|
916
|
+
# Fail closed: the step dials the URL itself, so without the gate
|
|
917
|
+
# nothing would apply the flow's network policy.
|
|
918
|
+
return HttpResponse(
|
|
919
|
+
HandlerResult(
|
|
920
|
+
success=False,
|
|
921
|
+
error=(
|
|
922
|
+
"this platform cannot apply the flow's network policy to the request, "
|
|
923
|
+
"so it was not sent"
|
|
924
|
+
),
|
|
925
|
+
summary="Config error",
|
|
926
|
+
)
|
|
927
|
+
)
|
|
928
|
+
try:
|
|
929
|
+
credentials = await self._credentials(services, config)
|
|
930
|
+
verify = await self._tls_verify(services, config)
|
|
931
|
+
body = _json_body(config)
|
|
932
|
+
except ValueError as exc:
|
|
933
|
+
return HttpResponse(
|
|
934
|
+
HandlerResult(
|
|
935
|
+
success=False,
|
|
936
|
+
error=f"Invalid {HANDLER_ID} config: {exc}",
|
|
937
|
+
summary="Config error",
|
|
938
|
+
)
|
|
939
|
+
)
|
|
940
|
+
except RuntimeError as exc:
|
|
941
|
+
return HttpResponse(
|
|
942
|
+
HandlerResult(
|
|
943
|
+
success=False,
|
|
944
|
+
error=f"Could not load the organization's trusted CAs: {exc}",
|
|
945
|
+
summary="Trusted CAs unavailable",
|
|
946
|
+
)
|
|
947
|
+
)
|
|
948
|
+
return await self._run(services, config, credentials, verify, body)
|
|
949
|
+
|
|
950
|
+
@staticmethod
|
|
951
|
+
async def _credentials(services: HandlerServices, config: HttpRequestConfig) -> _Credentials:
|
|
952
|
+
"""Resolve Authentication into the header it sets. Raises ``ValueError``."""
|
|
953
|
+
header = _credential_header_name(config)
|
|
954
|
+
if header is None:
|
|
955
|
+
return _Credentials()
|
|
956
|
+
secret = await resolve_secret_text(
|
|
957
|
+
services,
|
|
958
|
+
config.auth_secret,
|
|
959
|
+
field="auth_secret",
|
|
960
|
+
source=f"{HANDLER_ID}.auth_secret",
|
|
961
|
+
)
|
|
962
|
+
if len(secret) < MIN_SCRUB_LENGTH:
|
|
963
|
+
raise ValueError(
|
|
964
|
+
f"auth_secret resolved to a value shorter than {MIN_SCRUB_LENGTH} characters, "
|
|
965
|
+
"too short to keep reliably out of the step's output; use a longer secret"
|
|
966
|
+
)
|
|
967
|
+
# A basic-auth password is base64-encoded; any other secret is sent as is.
|
|
968
|
+
if config.auth_type != "basic" and _header_value_error("auth_secret", secret):
|
|
969
|
+
raise ValueError(
|
|
970
|
+
"auth_secret resolved to a value with line breaks, control characters or "
|
|
971
|
+
"non-ASCII characters, which cannot be sent in a header"
|
|
972
|
+
)
|
|
973
|
+
if config.auth_type == "bearer":
|
|
974
|
+
value = f"Bearer {secret}"
|
|
975
|
+
extra: tuple[str, ...] = ()
|
|
976
|
+
elif config.auth_type == "basic":
|
|
977
|
+
token = base64.b64encode(f"{config.auth_username}:{secret}".encode()).decode()
|
|
978
|
+
value = f"Basic {token}"
|
|
979
|
+
extra = (token,)
|
|
980
|
+
else:
|
|
981
|
+
value = f"{config.auth_prefix}{secret}"
|
|
982
|
+
extra = ()
|
|
983
|
+
return _Credentials(header=header, value=value, secrets=(secret, value, *extra))
|
|
984
|
+
|
|
985
|
+
@staticmethod
|
|
986
|
+
async def _tls_verify(services: HandlerServices, config: HttpRequestConfig) -> Any:
|
|
987
|
+
"""``verify`` for the client: False, True, or a context trusting the CA bundle.
|
|
988
|
+
|
|
989
|
+
Without a CA bundle the client trusts its default store plus the run
|
|
990
|
+
organization's trusted CAs (just the default store when it has none,
|
|
991
|
+
or on a host without the service). A CA bundle replaces both: only
|
|
992
|
+
its CAs are trusted. Raises ``RuntimeError`` when the host cannot
|
|
993
|
+
load the organization's list for an ``https://`` URL. A plain
|
|
994
|
+
``http://`` request is still sent then: its first hop needs no
|
|
995
|
+
trust, and an HTTPS hop it is redirected to trusts the default
|
|
996
|
+
store alone, which is stricter, never looser.
|
|
997
|
+
"""
|
|
998
|
+
if not config.verify_tls:
|
|
999
|
+
return False
|
|
1000
|
+
if not config.ca_bundle.strip():
|
|
1001
|
+
try:
|
|
1002
|
+
org_pem = await org_trusted_ca_pem(services)
|
|
1003
|
+
except RuntimeError:
|
|
1004
|
+
if httpx.URL(config.url).scheme == "https":
|
|
1005
|
+
raise
|
|
1006
|
+
return True
|
|
1007
|
+
if org_pem is None:
|
|
1008
|
+
return True
|
|
1009
|
+
try:
|
|
1010
|
+
return add_trusted_cas(httpx.create_ssl_context(trust_env=False), org_pem)
|
|
1011
|
+
except (ssl.SSLError, ValueError) as exc:
|
|
1012
|
+
raise RuntimeError(f"a trusted CA could not be loaded: {exc}") from None
|
|
1013
|
+
pem = await resolve_secret_text(
|
|
1014
|
+
services, config.ca_bundle, field="ca_bundle", source=f"{HANDLER_ID}.ca_bundle"
|
|
1015
|
+
)
|
|
1016
|
+
if _PEM_CERTIFICATE not in pem:
|
|
1017
|
+
raise ValueError("ca_bundle holds no PEM certificate (-----BEGIN CERTIFICATE-----)")
|
|
1018
|
+
try:
|
|
1019
|
+
return ssl.create_default_context(cadata=pem)
|
|
1020
|
+
except (ssl.SSLError, ValueError) as exc:
|
|
1021
|
+
raise ValueError(f"ca_bundle could not be loaded: {exc}") from None
|
|
1022
|
+
|
|
1023
|
+
async def _run(
|
|
1024
|
+
self,
|
|
1025
|
+
services: HandlerServices,
|
|
1026
|
+
config: HttpRequestConfig,
|
|
1027
|
+
credentials: _Credentials,
|
|
1028
|
+
verify: Any,
|
|
1029
|
+
json_body: tuple[bool, Any],
|
|
1030
|
+
) -> HttpResponse:
|
|
1031
|
+
"""Send the request (and next pages), then build the output, evidence and result."""
|
|
1032
|
+
scrub = Scrubber(credentials.secrets)
|
|
1033
|
+
ranges = parse_status_spec(config.expected_status)
|
|
1034
|
+
paginating = config.max_pages > 1
|
|
1035
|
+
next_path = parse_path(config.paginate_next) if paginating else ()
|
|
1036
|
+
items_path = parse_path(config.paginate_items) if paginating else ()
|
|
1037
|
+
limit_text = f"Max Response Size ({config.max_response_bytes} bytes)"
|
|
1038
|
+
if paginating:
|
|
1039
|
+
limit_text += " summed over the pages"
|
|
1040
|
+
|
|
1041
|
+
headers: dict[str, str] = dict(config.headers)
|
|
1042
|
+
lowered = {name.lower() for name in headers}
|
|
1043
|
+
content: bytes | None = None
|
|
1044
|
+
has_json, json_value = json_body
|
|
1045
|
+
if has_json:
|
|
1046
|
+
content = json.dumps(
|
|
1047
|
+
json_value, ensure_ascii=False, separators=(",", ":"), allow_nan=False
|
|
1048
|
+
).encode("utf-8")
|
|
1049
|
+
if "content-type" not in lowered:
|
|
1050
|
+
headers["Content-Type"] = "application/json"
|
|
1051
|
+
elif config.text_body:
|
|
1052
|
+
content = config.text_body.encode("utf-8")
|
|
1053
|
+
if "content-type" not in lowered:
|
|
1054
|
+
headers["Content-Type"] = "text/plain; charset=utf-8"
|
|
1055
|
+
if credentials.header:
|
|
1056
|
+
headers[credentials.header] = credentials.value
|
|
1057
|
+
|
|
1058
|
+
url = httpx.URL(config.url.strip())
|
|
1059
|
+
if config.query:
|
|
1060
|
+
url = url.copy_merge_params(config.query)
|
|
1061
|
+
display_url = str(url.copy_with(query=None, fragment=None))
|
|
1062
|
+
|
|
1063
|
+
enforce = getattr(services, "enforce_egress", None)
|
|
1064
|
+
exchanges: list[_Exchange] = []
|
|
1065
|
+
pages: list[_Page] = []
|
|
1066
|
+
|
|
1067
|
+
async def gate(request: httpx.Request) -> None:
|
|
1068
|
+
"""Every outgoing request, redirect hops and pages included, before it dials.
|
|
1069
|
+
|
|
1070
|
+
The host is checked in the ASCII form the connection dials (see
|
|
1071
|
+
``url_host``). The egress policy's denial propagates as raised (the
|
|
1072
|
+
step reports the policy's reason); the step's own refusals are
|
|
1073
|
+
RequestRefused.
|
|
1074
|
+
"""
|
|
1075
|
+
target = request.url
|
|
1076
|
+
if target.scheme not in SCHEME_PORTS:
|
|
1077
|
+
raise RequestRefused(
|
|
1078
|
+
f"refused to send a request to a {target.scheme!r} URL: only http and "
|
|
1079
|
+
"https are supported"
|
|
1080
|
+
)
|
|
1081
|
+
host = url_host(target)
|
|
1082
|
+
port = url_port(target)
|
|
1083
|
+
if enforce is not None:
|
|
1084
|
+
await enforce(host, port, proto="tcp", kind="http request")
|
|
1085
|
+
await check_destination(host, port)
|
|
1086
|
+
|
|
1087
|
+
async def observe(response: httpx.Response) -> None:
|
|
1088
|
+
"""Every response, before httpx reads its Location header.
|
|
1089
|
+
|
|
1090
|
+
The status and headers are recorded first, so a redirect that
|
|
1091
|
+
cannot be followed still reports them. Without Follow redirects the
|
|
1092
|
+
response is the result: its Location is kept from httpx, which
|
|
1093
|
+
would otherwise build a next request from it, and fail the step on
|
|
1094
|
+
one that is not a valid URL.
|
|
1095
|
+
"""
|
|
1096
|
+
exchange = exchanges[-1]
|
|
1097
|
+
exchange.status = response.status_code
|
|
1098
|
+
exchange.headers = filtered_response_headers(response.headers)
|
|
1099
|
+
if not response.has_redirect_location:
|
|
1100
|
+
return
|
|
1101
|
+
if not config.follow_redirects:
|
|
1102
|
+
del response.headers["location"]
|
|
1103
|
+
return
|
|
1104
|
+
try:
|
|
1105
|
+
httpx.URL(response.headers["location"])
|
|
1106
|
+
except httpx.InvalidURL as exc:
|
|
1107
|
+
raise RequestRefused(
|
|
1108
|
+
f"cannot follow the HTTP {response.status_code} redirect: its Location "
|
|
1109
|
+
f"header is not a valid URL ({exc})"
|
|
1110
|
+
) from None
|
|
1111
|
+
|
|
1112
|
+
client_options: dict[str, Any] = {
|
|
1113
|
+
"verify": verify,
|
|
1114
|
+
"trust_env": False,
|
|
1115
|
+
"follow_redirects": False,
|
|
1116
|
+
"timeout": httpx.Timeout(config.request_timeout_seconds),
|
|
1117
|
+
"event_hooks": {"request": [gate], "response": [observe]},
|
|
1118
|
+
# Only what _read_body can decode within the size budget.
|
|
1119
|
+
"headers": {"Accept-Encoding": ACCEPT_ENCODING},
|
|
1120
|
+
}
|
|
1121
|
+
if self._transport is not None:
|
|
1122
|
+
client_options["transport"] = self._transport
|
|
1123
|
+
|
|
1124
|
+
sent_header_names: list[str] = []
|
|
1125
|
+
combined: Any = None
|
|
1126
|
+
items: list[Any] = []
|
|
1127
|
+
more_pages = False
|
|
1128
|
+
failure: str | None = None
|
|
1129
|
+
transient = False
|
|
1130
|
+
started = time.perf_counter()
|
|
1131
|
+
|
|
1132
|
+
async with httpx.AsyncClient(**client_options) as client:
|
|
1133
|
+
request = client.build_request(config.method, url, headers=headers, content=content)
|
|
1134
|
+
origin = request.url
|
|
1135
|
+
sent_header_names = sorted({name.lower() for name in request.headers})
|
|
1136
|
+
fetched = {str(origin)}
|
|
1137
|
+
try:
|
|
1138
|
+
while True:
|
|
1139
|
+
used = sum(len(page.body) for page in pages)
|
|
1140
|
+
page = await self._fetch(
|
|
1141
|
+
client,
|
|
1142
|
+
request,
|
|
1143
|
+
config=config,
|
|
1144
|
+
exchanges=exchanges,
|
|
1145
|
+
remaining=config.max_response_bytes - used,
|
|
1146
|
+
limit_text=limit_text,
|
|
1147
|
+
credential_header=credentials.header,
|
|
1148
|
+
has_body=content is not None,
|
|
1149
|
+
stay_at=origin if paginating else None,
|
|
1150
|
+
)
|
|
1151
|
+
pages.append(page)
|
|
1152
|
+
status = page.exchange.status or 0
|
|
1153
|
+
if not status_matches(status, ranges):
|
|
1154
|
+
where = f"page {len(pages)} answered " if len(pages) > 1 else ""
|
|
1155
|
+
failure = (
|
|
1156
|
+
f"{where}HTTP {status} is not in expected_status "
|
|
1157
|
+
f"{config.expected_status}"
|
|
1158
|
+
)
|
|
1159
|
+
break
|
|
1160
|
+
if not paginating:
|
|
1161
|
+
break
|
|
1162
|
+
parsed, data = page.json
|
|
1163
|
+
if not parsed:
|
|
1164
|
+
failure = f"page {len(pages)} is not JSON, so it cannot be paginated"
|
|
1165
|
+
break
|
|
1166
|
+
page_items = get_path(data, items_path)
|
|
1167
|
+
if not isinstance(page_items, list):
|
|
1168
|
+
failure = (
|
|
1169
|
+
f"page {len(pages)} has no list at paginate_items "
|
|
1170
|
+
f"{config.paginate_items!r}"
|
|
1171
|
+
)
|
|
1172
|
+
break
|
|
1173
|
+
items.extend(page_items)
|
|
1174
|
+
if len(pages) == 1:
|
|
1175
|
+
combined = data
|
|
1176
|
+
next_link = get_path(data, next_path)
|
|
1177
|
+
if isinstance(next_link, str):
|
|
1178
|
+
next_link = next_link.strip()
|
|
1179
|
+
if next_link is None or next_link == "":
|
|
1180
|
+
break
|
|
1181
|
+
if not isinstance(next_link, str):
|
|
1182
|
+
failure = (
|
|
1183
|
+
f"page {len(pages)} has a {type(next_link).__name__} at "
|
|
1184
|
+
f"paginate_next {config.paginate_next!r}, not a URL"
|
|
1185
|
+
)
|
|
1186
|
+
break
|
|
1187
|
+
if len(pages) >= config.max_pages:
|
|
1188
|
+
more_pages = True
|
|
1189
|
+
break
|
|
1190
|
+
try:
|
|
1191
|
+
next_url = page.url.join(next_link)
|
|
1192
|
+
except (httpx.InvalidURL, ValueError) as exc:
|
|
1193
|
+
failure = f"page {len(pages)} has an invalid next-page URL: {exc}"
|
|
1194
|
+
break
|
|
1195
|
+
if not same_origin(next_url, origin):
|
|
1196
|
+
failure = (
|
|
1197
|
+
f"refused the next-page URL of page {len(pages)}: it leaves "
|
|
1198
|
+
f"{origin.scheme}://{origin.netloc.decode('ascii')} (next pages must "
|
|
1199
|
+
"keep the first request's scheme, host and port)"
|
|
1200
|
+
)
|
|
1201
|
+
break
|
|
1202
|
+
if str(next_url) in fetched:
|
|
1203
|
+
failure = (
|
|
1204
|
+
f"page {len(pages)} links back to a page already fetched, so "
|
|
1205
|
+
"pagination stopped"
|
|
1206
|
+
)
|
|
1207
|
+
break
|
|
1208
|
+
fetched.add(str(next_url))
|
|
1209
|
+
request = client.build_request("GET", next_url, headers=headers)
|
|
1210
|
+
except (
|
|
1211
|
+
RequestRefused,
|
|
1212
|
+
_BodyError,
|
|
1213
|
+
httpx.HTTPError,
|
|
1214
|
+
httpx.InvalidURL,
|
|
1215
|
+
TimeoutError,
|
|
1216
|
+
) as exc:
|
|
1217
|
+
failure = _describe_error(exc, config.request_timeout_seconds)
|
|
1218
|
+
transient = _is_transient(exc)
|
|
1219
|
+
|
|
1220
|
+
elapsed_ms = _elapsed_ms(started)
|
|
1221
|
+
last = exchanges[-1] if exchanges else None
|
|
1222
|
+
last_page = pages[-1] if pages and last is not None and pages[-1].exchange is last else None
|
|
1223
|
+
status = last.status if last is not None else None
|
|
1224
|
+
# A response with the expected status is not ok when the step still
|
|
1225
|
+
# failed on it (its body, a redirect, a page).
|
|
1226
|
+
ok = failure is None and status is not None and status_matches(status, ranges)
|
|
1227
|
+
size_bytes = sum(len(page.body) for page in pages)
|
|
1228
|
+
|
|
1229
|
+
if last_page is not None:
|
|
1230
|
+
# Redacted whole, before any preview is cut from it: a credential
|
|
1231
|
+
# crossing the cut would otherwise survive in part.
|
|
1232
|
+
text = scrub.text(last_page.text)
|
|
1233
|
+
parsed, data = last_page.json
|
|
1234
|
+
if not parsed:
|
|
1235
|
+
data = None
|
|
1236
|
+
body_sha256: str | None = hashlib.sha256(last_page.body).hexdigest()
|
|
1237
|
+
else:
|
|
1238
|
+
text, data, body_sha256 = "", None, None
|
|
1239
|
+
if paginating and failure is None and combined is not None:
|
|
1240
|
+
data = replace_at(combined, items_path, items)
|
|
1241
|
+
|
|
1242
|
+
output: dict[str, Any] = {
|
|
1243
|
+
"status": status,
|
|
1244
|
+
"ok": ok,
|
|
1245
|
+
"url": last.url if last is not None else str(url),
|
|
1246
|
+
"method": config.method,
|
|
1247
|
+
"headers": dict(last.headers) if last is not None else {},
|
|
1248
|
+
"elapsed_ms": elapsed_ms,
|
|
1249
|
+
"size_bytes": size_bytes,
|
|
1250
|
+
"body_sha256": body_sha256,
|
|
1251
|
+
"pages": len(pages),
|
|
1252
|
+
"more_pages": more_pages,
|
|
1253
|
+
}
|
|
1254
|
+
meta: dict[str, Any] = {
|
|
1255
|
+
"method": config.method,
|
|
1256
|
+
"url": exchanges[0].url if exchanges else str(url),
|
|
1257
|
+
"status": status,
|
|
1258
|
+
"elapsed_ms": elapsed_ms,
|
|
1259
|
+
"request_headers": sent_header_names,
|
|
1260
|
+
"response_headers": dict(output["headers"]),
|
|
1261
|
+
"requests": [exchange.record() for exchange in exchanges],
|
|
1262
|
+
"pages": len(pages),
|
|
1263
|
+
"size_bytes": size_bytes,
|
|
1264
|
+
"body_sha256": body_sha256,
|
|
1265
|
+
}
|
|
1266
|
+
if not config.sensitive:
|
|
1267
|
+
data_truncated = data is not None and _json_size(data) > MAX_DATA_OUTPUT_BYTES
|
|
1268
|
+
output["body"] = text[:MAX_BODY_OUTPUT_CHARS]
|
|
1269
|
+
output["body_truncated"] = len(text) > MAX_BODY_OUTPUT_CHARS
|
|
1270
|
+
output["data"] = None if data_truncated else data
|
|
1271
|
+
output["data_truncated"] = data_truncated
|
|
1272
|
+
# Extract reads the whole response, however large its data.
|
|
1273
|
+
output["extracted"] = {
|
|
1274
|
+
name: get_path(data, parse_path(path)) for name, path in config.extract.items()
|
|
1275
|
+
}
|
|
1276
|
+
meta["body_preview"] = text[:EVIDENCE_PREVIEW_CHARS]
|
|
1277
|
+
meta["body_truncated"] = len(text) > EVIDENCE_PREVIEW_CHARS
|
|
1278
|
+
if failure:
|
|
1279
|
+
meta["error"] = failure
|
|
1280
|
+
|
|
1281
|
+
timing = f"{len(pages)} pages, {elapsed_ms} ms" if len(pages) > 1 else f"{elapsed_ms} ms"
|
|
1282
|
+
if status is None:
|
|
1283
|
+
summary = f"{config.method} {display_url} failed ({timing})"
|
|
1284
|
+
elif failure:
|
|
1285
|
+
summary = f"{config.method} {display_url} -> {status}, failed ({timing})"
|
|
1286
|
+
else:
|
|
1287
|
+
summary = f"{config.method} {display_url} -> {status} ({timing})"
|
|
1288
|
+
|
|
1289
|
+
metrics: dict[str, Any] = {"elapsed_ms": elapsed_ms, "size_bytes": size_bytes, "ok": ok}
|
|
1290
|
+
if status is not None:
|
|
1291
|
+
metrics["status"] = status
|
|
1292
|
+
evidence = [
|
|
1293
|
+
{"kind": "transport_meta", "name": "http_request", "content_json": scrub.tree(meta)}
|
|
1294
|
+
]
|
|
1295
|
+
result = HandlerResult(
|
|
1296
|
+
success=failure is None,
|
|
1297
|
+
summary=scrub.text(summary),
|
|
1298
|
+
metrics=metrics,
|
|
1299
|
+
evidence=evidence,
|
|
1300
|
+
error=scrub.text(failure) if failure else None,
|
|
1301
|
+
output=scrub.tree(output),
|
|
1302
|
+
)
|
|
1303
|
+
return HttpResponse(result, None if config.sensitive else scrub.tree(data), transient)
|
|
1304
|
+
|
|
1305
|
+
@staticmethod
|
|
1306
|
+
async def _fetch(
|
|
1307
|
+
client: httpx.AsyncClient,
|
|
1308
|
+
request: httpx.Request,
|
|
1309
|
+
*,
|
|
1310
|
+
config: HttpRequestConfig,
|
|
1311
|
+
exchanges: list[_Exchange],
|
|
1312
|
+
remaining: int,
|
|
1313
|
+
limit_text: str,
|
|
1314
|
+
credential_header: str | None,
|
|
1315
|
+
has_body: bool = False,
|
|
1316
|
+
stay_at: httpx.URL | None = None,
|
|
1317
|
+
) -> _Page:
|
|
1318
|
+
"""Send ``request``, following redirects when configured; the final response, read.
|
|
1319
|
+
|
|
1320
|
+
A redirect that would repeat the request body (``has_body``) to another
|
|
1321
|
+
scheme, host or port is refused. With ``stay_at`` (while paginating), a
|
|
1322
|
+
redirect away from that URL's scheme, host and port is refused, as a
|
|
1323
|
+
next-page link would be.
|
|
1324
|
+
"""
|
|
1325
|
+
hops = 0
|
|
1326
|
+
while True:
|
|
1327
|
+
exchange = _Exchange(method=request.method, url=str(request.url))
|
|
1328
|
+
exchanges.append(exchange)
|
|
1329
|
+
started = time.perf_counter()
|
|
1330
|
+
next_request: httpx.Request | None = None
|
|
1331
|
+
try:
|
|
1332
|
+
async with asyncio.timeout(config.request_timeout_seconds):
|
|
1333
|
+
# The client's response hook records the status and headers.
|
|
1334
|
+
response = await client.send(request, stream=True)
|
|
1335
|
+
try:
|
|
1336
|
+
if config.follow_redirects and response.next_request is not None:
|
|
1337
|
+
# The redirect's own body is never read.
|
|
1338
|
+
next_request = response.next_request
|
|
1339
|
+
else:
|
|
1340
|
+
body = await _read_body(response, remaining, limit_text)
|
|
1341
|
+
return _Page(
|
|
1342
|
+
exchange=exchange,
|
|
1343
|
+
url=response.url,
|
|
1344
|
+
body=body,
|
|
1345
|
+
charset=response.charset_encoding,
|
|
1346
|
+
)
|
|
1347
|
+
finally:
|
|
1348
|
+
await response.aclose()
|
|
1349
|
+
finally:
|
|
1350
|
+
exchange.elapsed_ms = _elapsed_ms(started)
|
|
1351
|
+
if next_request is None: # pragma: no cover - every path above returns or sets it
|
|
1352
|
+
raise RequestRefused("the redirect could not be followed")
|
|
1353
|
+
hops += 1
|
|
1354
|
+
if hops > MAX_REDIRECTS:
|
|
1355
|
+
raise RequestRefused(f"stopped after {MAX_REDIRECTS} redirects")
|
|
1356
|
+
if next_request.url.scheme not in SCHEME_PORTS:
|
|
1357
|
+
raise RequestRefused(
|
|
1358
|
+
f"refused to follow a redirect to a {next_request.url.scheme!r} URL: only "
|
|
1359
|
+
"http and https are supported"
|
|
1360
|
+
)
|
|
1361
|
+
if stay_at is not None and not same_origin(next_request.url, stay_at):
|
|
1362
|
+
raise RequestRefused(
|
|
1363
|
+
f"refused a redirect to {next_request.url.scheme}://"
|
|
1364
|
+
f"{next_request.url.netloc.decode('ascii')} while paginating (every page "
|
|
1365
|
+
"must keep the first request's scheme, host and port)"
|
|
1366
|
+
)
|
|
1367
|
+
# httpx keeps the method and body except where the redirect turns it into a GET.
|
|
1368
|
+
has_body = has_body and next_request.method == request.method
|
|
1369
|
+
if not same_origin(next_request.url, request.url):
|
|
1370
|
+
if has_body:
|
|
1371
|
+
raise RequestRefused(
|
|
1372
|
+
f"refused to follow the HTTP {exchange.status} redirect to "
|
|
1373
|
+
f"{next_request.url.scheme}://{next_request.url.netloc.decode('ascii')}: "
|
|
1374
|
+
"it would send the request body to another scheme, host or port; "
|
|
1375
|
+
"send the request to that URL instead"
|
|
1376
|
+
)
|
|
1377
|
+
_strip_for_new_origin(next_request.headers, config.headers, credential_header)
|
|
1378
|
+
request = next_request
|
|
1379
|
+
|
|
1380
|
+
|
|
1381
|
+
def _json_size(value: Any) -> int:
|
|
1382
|
+
"""Bytes of ``value`` as compact UTF-8 JSON."""
|
|
1383
|
+
text = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
1384
|
+
return len(text.encode("utf-8", errors="surrogatepass"))
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
#: Failures where the request may not have reached the server (see HttpResponse.transient).
|
|
1388
|
+
_TRANSIENT_ERRORS = (
|
|
1389
|
+
TimeoutError,
|
|
1390
|
+
httpx.TimeoutException,
|
|
1391
|
+
httpx.ConnectError,
|
|
1392
|
+
httpx.ReadError,
|
|
1393
|
+
httpx.WriteError,
|
|
1394
|
+
httpx.RemoteProtocolError,
|
|
1395
|
+
)
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
#: TLS errors that are a dropped connection rather than a handshake the server
|
|
1399
|
+
#: refused: the peer went away, which the next attempt may not meet again.
|
|
1400
|
+
_DROPPED_TLS_ERRORS = (ssl.SSLEOFError, ssl.SSLSyscallError, ssl.SSLZeroReturnError)
|
|
1401
|
+
_DROPPED_TLS_REASONS = frozenset({"UNEXPECTED_EOF_WHILE_READING"})
|
|
1402
|
+
|
|
1403
|
+
|
|
1404
|
+
def _is_transient(exc: BaseException) -> bool:
|
|
1405
|
+
"""Whether the request may not have reached the server (see ``HttpResponse.transient``).
|
|
1406
|
+
|
|
1407
|
+
A connection that failed in the TLS handshake (a certificate the step does
|
|
1408
|
+
not trust, a name mismatch, a protocol the server refuses) is not: the
|
|
1409
|
+
next attempt meets the same server. httpx raises that ``ssl.SSLError`` as
|
|
1410
|
+
a ``ConnectError`` from httpcore's, which re-raises it ``from None``, so
|
|
1411
|
+
the error is found through ``__context__`` where ``__cause__`` is unset.
|
|
1412
|
+
A connection the peer dropped during the handshake stays transient.
|
|
1413
|
+
"""
|
|
1414
|
+
if not isinstance(exc, _TRANSIENT_ERRORS):
|
|
1415
|
+
return False
|
|
1416
|
+
seen: set[int] = set()
|
|
1417
|
+
cause: BaseException | None = exc
|
|
1418
|
+
while cause is not None and id(cause) not in seen and len(seen) < 8:
|
|
1419
|
+
seen.add(id(cause))
|
|
1420
|
+
if isinstance(cause, ssl.SSLError) and not _is_dropped_tls(cause):
|
|
1421
|
+
return False
|
|
1422
|
+
cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__
|
|
1423
|
+
return True
|
|
1424
|
+
|
|
1425
|
+
|
|
1426
|
+
def _is_dropped_tls(exc: ssl.SSLError) -> bool:
|
|
1427
|
+
"""Whether ``exc`` says the peer went away, not that it refused the handshake."""
|
|
1428
|
+
if isinstance(exc, _DROPPED_TLS_ERRORS):
|
|
1429
|
+
return True
|
|
1430
|
+
return getattr(exc, "reason", None) in _DROPPED_TLS_REASONS
|
|
1431
|
+
|
|
1432
|
+
|
|
1433
|
+
def _describe_error(exc: BaseException, timeout_seconds: int) -> str:
|
|
1434
|
+
"""A step error for a request that failed before a usable response."""
|
|
1435
|
+
if isinstance(exc, (RequestRefused, _BodyError)):
|
|
1436
|
+
return str(exc)
|
|
1437
|
+
if isinstance(exc, (TimeoutError, httpx.TimeoutException)):
|
|
1438
|
+
return f"the request timed out after {timeout_seconds} s"
|
|
1439
|
+
detail = str(exc).strip()
|
|
1440
|
+
name = type(exc).__name__
|
|
1441
|
+
return f"request failed: {name}: {detail}" if detail else f"request failed: {name}"
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
__all__ = [
|
|
1445
|
+
"HANDLER_ID",
|
|
1446
|
+
"HttpRequestConfig",
|
|
1447
|
+
"HttpRequestHandler",
|
|
1448
|
+
"HttpResponse",
|
|
1449
|
+
"config_errors",
|
|
1450
|
+
"parse_status_spec",
|
|
1451
|
+
"status_matches",
|
|
1452
|
+
]
|