hegemony-steps-http 0.46.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1452 @@
1
+ # SPDX-FileCopyrightText: 2025-2026 Rexonix s.r.o.
2
+ # SPDX-FileContributor: Jakub Trávník <jakub.travnik@gmail.com>
3
+ #
4
+ # SPDX-License-Identifier: AGPL-3.0-or-later
5
+
6
+ """HttpRequestHandler: send one HTTP(S) request and return the response as step output.
7
+
8
+ The worker sends the request itself, through its own httpx client, so every
9
+ outgoing request (redirect hops and next pages included) walks the flow's
10
+ egress gate (``services.enforce_egress``) and the step's own destination
11
+ check before a connection is opened. Redirects are followed here rather
12
+ than by httpx, so a redirect body is never read and the credentials and
13
+ author-set headers stay with the origin the author wrote.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import asyncio
19
+ import base64
20
+ import codecs
21
+ import hashlib
22
+ import json
23
+ import re
24
+ import ssl
25
+ import time
26
+ import zlib
27
+ from dataclasses import dataclass, field
28
+ from functools import cached_property
29
+ from typing import Any, Literal
30
+
31
+ import httpx
32
+ from pydantic import BaseModel, ConfigDict, Field
33
+
34
+ from hegemony_step_sdk import (
35
+ MAX_CHAIN_OUTPUT_CHARS,
36
+ BaseHandler,
37
+ HandlerContext,
38
+ HandlerResult,
39
+ HandlerServices,
40
+ HandlerTargeting,
41
+ InvalidConfigError,
42
+ StepKind,
43
+ )
44
+ from hegemony_step_sdk.trust import add_trusted_cas, org_trusted_ca_pem
45
+
46
+ from .jsonpath import PathError, get_path, parse_path, replace_at
47
+ from .security import (
48
+ MIN_SCRUB_LENGTH,
49
+ SCHEME_PORTS,
50
+ RequestRefused,
51
+ Scrubber,
52
+ check_destination,
53
+ filtered_response_headers,
54
+ is_bare_secret_ref,
55
+ is_templated,
56
+ resolve_secret_text,
57
+ same_origin,
58
+ url_host,
59
+ url_port,
60
+ )
61
+
62
+ HANDLER_ID = "http.request"
63
+
64
+ #: Methods that never carry a request body here.
65
+ BODYLESS_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
66
+
67
+ MIB = 1024 * 1024
68
+ #: Content encodings the step decodes itself, within the size budget; it asks
69
+ #: servers for nothing else.
70
+ ACCEPT_ENCODING = "gzip, deflate"
71
+ _GZIP_ENCODINGS = frozenset({"gzip", "x-gzip"})
72
+ _IDENTITY_ENCODINGS = frozenset({"", "identity"})
73
+ #: Raw bytes a compressed body may take beyond the decoded budget (headers,
74
+ #: trailers and incompressible blocks), so a stream that expands to nothing
75
+ #: cannot be read forever.
76
+ _RAW_SLACK_BYTES = MIB
77
+ DEFAULT_MAX_RESPONSE_BYTES = MIB
78
+ MAX_RESPONSE_BYTES_LIMIT = 16 * MIB
79
+ MAX_PAGES_LIMIT = 100
80
+ MAX_TIMEOUT_SECONDS = 600
81
+ #: Redirect hops followed per request before the step gives up.
82
+ MAX_REDIRECTS = 10
83
+ #: Characters of the last response body kept in ``output.body``.
84
+ MAX_BODY_OUTPUT_CHARS = 64 * 1024
85
+ #: Largest ``output.data``, in bytes of compact JSON. Step outputs are chained
86
+ #: into later steps' templates and snapshots, so they stay small: larger data
87
+ #: is left out (``data_truncated``), while Extract still reads all of it.
88
+ MAX_DATA_OUTPUT_BYTES = 64 * 1024
89
+ #: Characters of the last response body kept in the evidence record.
90
+ EVIDENCE_PREVIEW_CHARS = MAX_CHAIN_OUTPUT_CHARS
91
+
92
+ #: A header name (an RFC 9110 token).
93
+ _HEADER_NAME_RE = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$")
94
+ #: Control characters a header value may not hold (tab is allowed).
95
+ _HEADER_VALUE_BAD_RE = re.compile(r"[\x00-\x08\x0a-\x1f\x7f]")
96
+ #: Headers the step sets itself (framing, routing): never from config.
97
+ _RESERVED_HEADERS = frozenset({"host", "content-length", "transfer-encoding", "connection"})
98
+ #: Author-set headers a redirect to another origin keeps. Every other header
99
+ #: the author set, and the credentials, stay with the original origin.
100
+ _CROSS_ORIGIN_SAFE_HEADERS = frozenset({"accept", "accept-language", "content-type", "user-agent"})
101
+ #: Names of extracted values (``steps.<id>.output.extracted.<name>``).
102
+ _EXTRACT_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$")
103
+ _PEM_CERTIFICATE = "-----BEGIN CERTIFICATE-----"
104
+ _LITERAL_SCHEME_RE = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*)://")
105
+
106
+
107
+ def parse_status_spec(spec: str) -> list[tuple[int, int]]:
108
+ """Parse an expected-status spec like ``200-299,304`` into ranges.
109
+
110
+ Raises ``ValueError`` for malformed entries.
111
+ """
112
+ ranges: list[tuple[int, int]] = []
113
+ for raw in spec.split(","):
114
+ entry = raw.strip()
115
+ if not entry:
116
+ continue
117
+ try:
118
+ if "-" in entry:
119
+ low_raw, high_raw = entry.split("-", 1)
120
+ low, high = int(low_raw), int(high_raw)
121
+ else:
122
+ low = high = int(entry)
123
+ except ValueError:
124
+ raise ValueError(f"invalid status entry: {entry}") from None
125
+ if not (100 <= low <= 599 and 100 <= high <= 599 and low <= high):
126
+ raise ValueError(f"invalid status entry: {entry}")
127
+ ranges.append((low, high))
128
+ if not ranges:
129
+ raise ValueError("expected_status must list at least one status or range")
130
+ return ranges
131
+
132
+
133
+ def status_matches(status: int, ranges: list[tuple[int, int]]) -> bool:
134
+ """Whether ``status`` falls in one of ``ranges``."""
135
+ return any(low <= status <= high for low, high in ranges)
136
+
137
+
138
+ class HttpRequestConfig(BaseModel):
139
+ """Config for ``http.request``."""
140
+
141
+ # A number written for a text setting (expected_status: 201, a query or
142
+ # header value) is its text.
143
+ model_config = ConfigDict(extra="allow", coerce_numbers_to_str=True)
144
+
145
+ url: str = Field(
146
+ min_length=1,
147
+ title="URL",
148
+ description=(
149
+ "Address the request goes to, starting with https:// or http://, such as "
150
+ "https://api.example.com/v1/items. May use templates such as "
151
+ "{{ inputs.base_url }}/items. Credentials never go in the URL: use "
152
+ "Authentication."
153
+ ),
154
+ json_schema_extra={"x_placeholder": "https://api.example.com/v1/items", "x_col_span": 2},
155
+ )
156
+ method: Literal["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"] = Field(
157
+ default="GET",
158
+ title="Method",
159
+ description=(
160
+ "HTTP method of the request. GET, HEAD and OPTIONS send no body. Defaults to GET."
161
+ ),
162
+ )
163
+ query: dict[str, str] = Field(
164
+ default_factory=dict,
165
+ title="Query Parameters",
166
+ description=(
167
+ "Query parameters added to the URL, NAME=value, one per line. A parameter "
168
+ "the URL already carries is replaced. Values may use templates."
169
+ ),
170
+ json_schema_extra={
171
+ "x_widget": "env-vars",
172
+ "x_placeholder": "limit=100\nstatus=active",
173
+ "x_rows": 2,
174
+ "x_col_span": 2,
175
+ },
176
+ )
177
+ headers: dict[str, str] = Field(
178
+ default_factory=dict,
179
+ title="Headers",
180
+ description=(
181
+ "Request headers, Name=value, one per line, such as Accept=application/json. "
182
+ "Values may use templates. Put credentials in Authentication instead: a "
183
+ "redirect to another origin drops the credentials and every header set here "
184
+ "except Accept, Accept-Language, Content-Type and User-Agent. Host, "
185
+ "Content-Length, Transfer-Encoding and Connection are set by the step."
186
+ ),
187
+ json_schema_extra={
188
+ "x_widget": "env-vars",
189
+ "x_placeholder": "Accept=application/json",
190
+ "x_rows": 2,
191
+ "x_col_span": 2,
192
+ },
193
+ )
194
+ json_body: Any = Field(
195
+ default=None,
196
+ title="JSON Body",
197
+ description=(
198
+ "Request body sent as JSON, with Content-Type application/json unless "
199
+ "Headers set another. Text is parsed as JSON, so a template that renders "
200
+ 'JSON works, such as {"name": "{{ inputs.name }}"}. Not allowed with '
201
+ "GET, HEAD or OPTIONS, or together with Text Body."
202
+ ),
203
+ json_schema_extra={
204
+ "x_placeholder": '{"name": "{{ inputs.name }}"}',
205
+ "x_rows": 6,
206
+ "x_col_span": 2,
207
+ },
208
+ )
209
+ text_body: str = Field(
210
+ default="",
211
+ title="Text Body",
212
+ description=(
213
+ "Request body sent as written, with Content-Type text/plain; charset=utf-8 "
214
+ "unless Headers set another. Not allowed with GET, HEAD or OPTIONS, or "
215
+ "together with JSON Body."
216
+ ),
217
+ json_schema_extra={"x_rows": 4, "x_col_span": 2, "x_advanced": True},
218
+ )
219
+ auth_type: Literal["none", "bearer", "basic", "header"] = Field(
220
+ default="none",
221
+ title="Authentication",
222
+ description=(
223
+ "How the request authenticates: none; a bearer token (Authorization: Bearer "
224
+ "and the secret); basic, with a username and the secret as password; or a "
225
+ "custom header carrying the secret. Credentials go only to the URL's own "
226
+ "scheme, host and port: a redirect elsewhere drops them. They need an "
227
+ "https:// URL unless Allow credentials over HTTP is on."
228
+ ),
229
+ json_schema_extra={
230
+ "x_option_labels": {
231
+ "none": "None",
232
+ "bearer": "Bearer token",
233
+ "basic": "Basic (username and password)",
234
+ "header": "Custom header",
235
+ }
236
+ },
237
+ )
238
+ auth_secret: str = Field(
239
+ default="",
240
+ title="Secret",
241
+ description=(
242
+ "The token or password: a secret reference such as "
243
+ "vault://orgs/acme/secrets/api/token, a {{ secret('...') }} template, or "
244
+ "literal text. A value resolved from a reference is redacted from the run's "
245
+ "logs; literal text is not, so prefer a reference."
246
+ ),
247
+ json_schema_extra={
248
+ "x_placeholder": "vault://orgs/acme/secrets/api/token",
249
+ "x_hide_when": {"field": "auth_type", "value": "none"},
250
+ "x_col_span": 2,
251
+ },
252
+ )
253
+ auth_username: str = Field(
254
+ default="",
255
+ title="Username",
256
+ description=(
257
+ "Username for basic authentication; it may not contain a colon. May use templates."
258
+ ),
259
+ json_schema_extra={"x_show_when": {"field": "auth_type", "value": "basic"}},
260
+ )
261
+ auth_header_name: str = Field(
262
+ default="Authorization",
263
+ title="Header Name",
264
+ description=(
265
+ "Header that carries the secret when Authentication is Custom header, such "
266
+ "as X-API-Key. Defaults to Authorization."
267
+ ),
268
+ json_schema_extra={
269
+ "x_placeholder": "X-API-Key",
270
+ "x_show_when": {"field": "auth_type", "value": "header"},
271
+ },
272
+ )
273
+ auth_prefix: str = Field(
274
+ default="",
275
+ title="Value Prefix",
276
+ description=(
277
+ "Text sent before the secret in the custom header, such as 'Token ' with its "
278
+ "trailing space. Empty sends the secret alone."
279
+ ),
280
+ json_schema_extra={
281
+ "x_placeholder": "Token ",
282
+ "x_show_when": {"field": "auth_type", "value": "header"},
283
+ },
284
+ )
285
+ allow_insecure_auth: bool = Field(
286
+ default=False,
287
+ title="Allow credentials over HTTP",
288
+ description=(
289
+ "Send the Authentication credential to a plain http:// URL. Off by default, "
290
+ "because over HTTP the token or password travels unencrypted and anyone on "
291
+ "the network path can read it; turn on only for an internal endpoint that "
292
+ "offers no HTTPS."
293
+ ),
294
+ json_schema_extra={"x_advanced": True},
295
+ )
296
+ expected_status: str = Field(
297
+ default="200-299",
298
+ title="Expected Status",
299
+ description=(
300
+ "Comma-separated status codes and ranges that count as success, such as "
301
+ "200-299,304. Defaults to 200-299. With pagination every page must match."
302
+ ),
303
+ json_schema_extra={"x_placeholder": "200-299"},
304
+ )
305
+ request_timeout_seconds: int = Field(
306
+ default=30,
307
+ ge=1,
308
+ le=MAX_TIMEOUT_SECONDS,
309
+ title="Timeout (sec)",
310
+ description=(
311
+ "Time limit for each request, in seconds: connecting, sending and reading the "
312
+ "whole response. Redirect hops and pages each get their own. Defaults to 30; "
313
+ "1 to 600."
314
+ ),
315
+ )
316
+ follow_redirects: bool = Field(
317
+ default=False,
318
+ title="Follow redirects",
319
+ description=(
320
+ "Follow 3xx redirects, at most 10, and report the final response. Off by "
321
+ "default, when the redirect response itself is the result. Every hop is "
322
+ "checked against the egress policy, and a redirect to a scheme other than "
323
+ "http or https, or one that would send the request body to another origin, "
324
+ "is refused."
325
+ ),
326
+ )
327
+ verify_tls: bool = Field(
328
+ default=True,
329
+ title="Verify TLS certificate",
330
+ description=(
331
+ "Validate the server's certificate on HTTPS requests. Turn off only for test "
332
+ "endpoints with self-signed certificates, and prefer CA Bundle. On by default."
333
+ ),
334
+ json_schema_extra={"x_advanced": True},
335
+ )
336
+ ca_bundle: str = Field(
337
+ default="",
338
+ title="CA Bundle",
339
+ description=(
340
+ "PEM certificates to trust for this request instead of the public certificate "
341
+ "authorities and the organization's trusted CAs, which pins them: the PEM text, "
342
+ "a secret reference, or a {{ secret('...') }} template. Empty trusts the public "
343
+ "authorities plus the organization's trusted CAs (Settings, Trusted CAs). "
344
+ "Ignored when Verify TLS certificate is off."
345
+ ),
346
+ json_schema_extra={
347
+ "x_placeholder": "-----BEGIN CERTIFICATE-----",
348
+ "x_show_when": {"field": "verify_tls", "value": True},
349
+ "x_rows": 4,
350
+ "x_col_span": 2,
351
+ "x_advanced": True,
352
+ },
353
+ )
354
+ max_response_bytes: int = Field(
355
+ default=DEFAULT_MAX_RESPONSE_BYTES,
356
+ ge=1,
357
+ le=MAX_RESPONSE_BYTES_LIMIT,
358
+ title="Max Response Size (bytes)",
359
+ description=(
360
+ "Largest response body the step reads, in bytes, summed over all pages. A "
361
+ "larger response fails the step instead of being cut short. Defaults to "
362
+ "1048576 (1 MiB); at most 16777216 (16 MiB)."
363
+ ),
364
+ json_schema_extra={"x_advanced": True},
365
+ )
366
+ extract: dict[str, str] = Field(
367
+ default_factory=dict,
368
+ title="Extract",
369
+ description=(
370
+ "Values to pick from the JSON response, NAME=path one per line, such as "
371
+ "id=results[0].id. A path is keys separated by dots, [n] for list items "
372
+ "(negative counts from the end), and an optional leading $. A missing value "
373
+ "is null. Later steps read them as steps.<id>.output.extracted.NAME."
374
+ ),
375
+ json_schema_extra={
376
+ "x_widget": "env-vars",
377
+ "x_placeholder": "id=results[0].id",
378
+ "x_rows": 2,
379
+ "x_col_span": 2,
380
+ },
381
+ )
382
+ sensitive: bool = Field(
383
+ default=False,
384
+ title="Sensitive response",
385
+ description=(
386
+ "Treat the response as secret: its body, data and extracted values stay out "
387
+ "of the output and evidence, which keep the status, headers, size and SHA-256 "
388
+ "of the body. Off by default."
389
+ ),
390
+ json_schema_extra={"x_advanced": True},
391
+ )
392
+ paginate_next: str = Field(
393
+ default="",
394
+ title="Next Page Path",
395
+ description=(
396
+ "Path of the next page's URL in each JSON response, such as next. A missing, "
397
+ "null or empty value ends pagination. Next pages are fetched with GET, the "
398
+ "same headers and credentials, and must stay on the first request's scheme, "
399
+ "host and port. Needs GET, Items Path, and Max Pages above 1."
400
+ ),
401
+ json_schema_extra={"x_placeholder": "next", "x_advanced": True},
402
+ )
403
+ paginate_items: str = Field(
404
+ default="",
405
+ title="Items Path",
406
+ description=(
407
+ "Path of the item list in each JSON response, such as results. The output's "
408
+ "data is the first page with this list replaced by the items of every page."
409
+ ),
410
+ json_schema_extra={"x_placeholder": "results", "x_advanced": True},
411
+ )
412
+ max_pages: int = Field(
413
+ default=1,
414
+ ge=1,
415
+ le=MAX_PAGES_LIMIT,
416
+ title="Max Pages",
417
+ description=(
418
+ "Most pages fetched, the first included. 1, the default, fetches a single "
419
+ "response; at most 100."
420
+ ),
421
+ json_schema_extra={"x_advanced": True},
422
+ )
423
+
424
+
425
+ def _url_error(url: str, *, allow_templates: bool) -> str | None:
426
+ """Why ``url`` cannot be requested, or None."""
427
+ text = url.strip()
428
+ if not text:
429
+ return "'url' is required"
430
+ if allow_templates and is_templated(text):
431
+ literal = _LITERAL_SCHEME_RE.match(text)
432
+ if literal and literal.group(1).lower() not in SCHEME_PORTS:
433
+ return "'url' must start with http:// or https://"
434
+ return None
435
+ try:
436
+ parsed = httpx.URL(text)
437
+ except (httpx.InvalidURL, TypeError, ValueError) as exc:
438
+ return f"'url' is not a valid URL: {exc}"
439
+ if parsed.scheme not in SCHEME_PORTS:
440
+ return "'url' must start with http:// or https://"
441
+ if not parsed.host:
442
+ return "'url' has no host"
443
+ if parsed.userinfo:
444
+ return "'url' must not carry credentials (user:password@); use Authentication instead"
445
+ return None
446
+
447
+
448
+ def _header_value_error(field_name: str, value: str) -> str | None:
449
+ """Why ``value`` cannot be sent as a header value, or None. Never echoes the value."""
450
+ if _HEADER_VALUE_BAD_RE.search(value):
451
+ return f"{field_name} contains line breaks or control characters"
452
+ if not value.isascii():
453
+ return f"{field_name} contains non-ASCII characters"
454
+ return None
455
+
456
+
457
+ def _has_json_body(config: HttpRequestConfig) -> bool:
458
+ """Whether JSON Body is set (blank text counts as unset)."""
459
+ body = config.json_body
460
+ return body is not None and not (isinstance(body, str) and not body.strip())
461
+
462
+
463
+ def _json_body(config: HttpRequestConfig) -> tuple[bool, Any]:
464
+ """``(present, value)`` of the JSON body; text is parsed. Raises ``ValueError``."""
465
+ if not _has_json_body(config):
466
+ return False, None
467
+ body = config.json_body
468
+ if isinstance(body, str):
469
+ try:
470
+ body = json.loads(body)
471
+ except (ValueError, RecursionError) as exc:
472
+ raise ValueError(f"'json_body' is not valid JSON: {exc}") from None
473
+ try:
474
+ json.dumps(body, allow_nan=False)
475
+ except (TypeError, ValueError) as exc:
476
+ raise ValueError(f"'json_body' cannot be sent as JSON: {exc}") from None
477
+ return True, body
478
+
479
+
480
+ def _paths_errors(config: HttpRequestConfig) -> list[str]:
481
+ """Errors in the Extract paths and the pagination paths (empty when valid)."""
482
+ errors: list[str] = []
483
+ for name, path in config.extract.items():
484
+ if not _EXTRACT_NAME_RE.match(name):
485
+ errors.append(
486
+ f"'extract' name {name!r} must start with a letter or underscore and use only "
487
+ "letters, digits and underscores (max 64)"
488
+ )
489
+ try:
490
+ parse_path(path)
491
+ except PathError as exc:
492
+ errors.append(f"'extract.{name}': {exc}")
493
+ for field_name in ("paginate_next", "paginate_items"):
494
+ value = getattr(config, field_name).strip()
495
+ if value:
496
+ try:
497
+ parse_path(value)
498
+ except PathError as exc:
499
+ errors.append(f"'{field_name}': {exc}")
500
+ return errors
501
+
502
+
503
+ def config_errors(config: HttpRequestConfig, *, allow_templates: bool) -> list[str]:
504
+ """Semantic errors a schema cannot express (empty when valid).
505
+
506
+ With ``allow_templates`` (saving a flow), values still holding template
507
+ syntax pass; the step checks them again once the host has rendered them.
508
+ Messages name fields, never the values of headers or secrets.
509
+ """
510
+ errors: list[str] = []
511
+
512
+ def unchecked(value: Any) -> bool:
513
+ """Whether ``value`` is left for the check after rendering."""
514
+ return allow_templates and is_templated(value)
515
+
516
+ error = _url_error(config.url, allow_templates=allow_templates)
517
+ if error:
518
+ errors.append(error)
519
+
520
+ if any(not name.strip() for name in config.query):
521
+ errors.append("'query' names must not be empty")
522
+
523
+ credential_header = _credential_header_name(config)
524
+ for name, value in config.headers.items():
525
+ if not _HEADER_NAME_RE.match(name):
526
+ errors.append(f"'headers' name {name!r} is not a valid header name")
527
+ continue
528
+ lowered = name.lower()
529
+ if lowered in _RESERVED_HEADERS:
530
+ errors.append(f"'headers' must not set {name}: the step sets it")
531
+ elif credential_header and lowered == credential_header.lower():
532
+ errors.append(
533
+ f"'headers' must not set {name}: Authentication sets it; remove it from Headers"
534
+ )
535
+ if not unchecked(value):
536
+ error = _header_value_error(f"'headers.{name}'", value)
537
+ if error:
538
+ errors.append(error)
539
+
540
+ has_json = _has_json_body(config)
541
+ if has_json and not unchecked(config.json_body):
542
+ try:
543
+ _json_body(config)
544
+ except ValueError as exc:
545
+ errors.append(str(exc))
546
+ has_text = bool(config.text_body)
547
+ if has_json and has_text:
548
+ errors.append("'json_body' and 'text_body' cannot both be set")
549
+ if (has_json or has_text) and config.method in BODYLESS_METHODS:
550
+ errors.append(f"a {config.method} request sends no body: clear JSON Body and Text Body")
551
+
552
+ if config.auth_type != "none" and not config.auth_secret.strip():
553
+ errors.append(f"'auth_secret' is required for {config.auth_type} authentication")
554
+ if (
555
+ config.auth_type != "none"
556
+ and not config.allow_insecure_auth
557
+ and not unchecked(config.url)
558
+ and config.url.strip().lower().startswith("http://")
559
+ ):
560
+ errors.append(
561
+ "Authentication needs an https:// URL: over http:// the credential travels "
562
+ "unencrypted (turn on 'allow_insecure_auth' for an internal endpoint without HTTPS)"
563
+ )
564
+ if config.auth_type == "basic":
565
+ username = config.auth_username
566
+ if not username.strip():
567
+ errors.append("'auth_username' is required for basic authentication")
568
+ elif not unchecked(username):
569
+ # Base64-encoded into the header, so any text but a colon or a
570
+ # control character will do.
571
+ if ":" in username:
572
+ errors.append("'auth_username' must not contain a colon")
573
+ if _HEADER_VALUE_BAD_RE.search(username):
574
+ errors.append("'auth_username' contains line breaks or control characters")
575
+ if config.auth_type == "header":
576
+ name = config.auth_header_name.strip()
577
+ if not _HEADER_NAME_RE.match(name):
578
+ errors.append("'auth_header_name' is not a valid header name")
579
+ elif name.lower() in _RESERVED_HEADERS:
580
+ errors.append(f"'auth_header_name' must not be {name}: the step sets it")
581
+ if not unchecked(config.auth_prefix):
582
+ error = _header_value_error("'auth_prefix'", config.auth_prefix)
583
+ if error:
584
+ errors.append(error)
585
+
586
+ ca_bundle = config.ca_bundle.strip()
587
+ if (
588
+ config.verify_tls
589
+ and ca_bundle
590
+ and not is_templated(ca_bundle)
591
+ and not is_bare_secret_ref(ca_bundle)
592
+ and _PEM_CERTIFICATE not in ca_bundle
593
+ ):
594
+ errors.append(
595
+ "'ca_bundle' must hold PEM certificates (-----BEGIN CERTIFICATE-----), a secret "
596
+ "reference, or a {{ secret('...') }} template"
597
+ )
598
+
599
+ if not unchecked(config.expected_status):
600
+ try:
601
+ parse_status_spec(config.expected_status)
602
+ except ValueError as exc:
603
+ errors.append(f"'expected_status': {exc}")
604
+
605
+ errors.extend(_paths_errors(config))
606
+ if config.sensitive and config.extract:
607
+ errors.append(
608
+ "'extract' has no effect on a sensitive response, whose values stay out of the "
609
+ "output: clear Extract or turn off Sensitive response"
610
+ )
611
+
612
+ paginate_next = config.paginate_next.strip()
613
+ paginate_items = config.paginate_items.strip()
614
+ if paginate_next or paginate_items or config.max_pages > 1:
615
+ if not paginate_next:
616
+ errors.append("pagination needs 'paginate_next', the path of the next page's URL")
617
+ if not paginate_items:
618
+ errors.append("pagination needs 'paginate_items', the path of each page's items")
619
+ if config.max_pages < 2:
620
+ errors.append("pagination needs 'max_pages' above 1 (1 fetches only the first page)")
621
+ if config.method != "GET":
622
+ errors.append("pagination needs the GET method")
623
+ return errors
624
+
625
+
626
+ def _credential_header_name(config: HttpRequestConfig) -> str | None:
627
+ """The request header Authentication sets, or None."""
628
+ if config.auth_type in {"bearer", "basic"}:
629
+ return "Authorization"
630
+ if config.auth_type == "header":
631
+ return config.auth_header_name.strip() or "Authorization"
632
+ return None
633
+
634
+
635
+ @dataclass(frozen=True)
636
+ class _Credentials:
637
+ """The request header Authentication sets, and the values to scrub from reports."""
638
+
639
+ header: str | None = None
640
+ value: str = field(default="", repr=False)
641
+ secrets: tuple[str, ...] = field(default=(), repr=False)
642
+
643
+
644
+ @dataclass(frozen=True)
645
+ class HttpResponse:
646
+ """What :meth:`HttpRequestHandler.send` produced.
647
+
648
+ ``result`` is what an ``http.request`` step reports. ``data`` is the whole
649
+ parsed JSON body of the final response (the pages combined, when
650
+ paginating), with known credentials scrubbed and without the size cap
651
+ ``output.data`` has; None when the body is not JSON, the response is
652
+ sensitive, or no response came. ``transient`` is true when no usable
653
+ response came because of a timeout or a dropped or refused connection,
654
+ so the request may not have reached the server and a read may be tried
655
+ again.
656
+ """
657
+
658
+ result: HandlerResult
659
+ data: Any = None
660
+ transient: bool = False
661
+
662
+
663
+ @dataclass
664
+ class _Exchange:
665
+ """One request the step sent (a redirect hop or a page) and the status it got."""
666
+
667
+ method: str
668
+ url: str
669
+ status: int | None = None
670
+ headers: dict[str, str] = field(default_factory=dict)
671
+ elapsed_ms: int = 0
672
+
673
+ def record(self) -> dict[str, Any]:
674
+ """The exchange as recorded in the evidence."""
675
+ return {
676
+ "method": self.method,
677
+ "url": self.url,
678
+ "status": self.status,
679
+ "elapsed_ms": self.elapsed_ms,
680
+ }
681
+
682
+
683
+ @dataclass
684
+ class _Page:
685
+ """The final response of one page, its body read."""
686
+
687
+ exchange: _Exchange
688
+ url: httpx.URL
689
+ body: bytes
690
+ charset: str | None
691
+
692
+ @cached_property
693
+ def text(self) -> str:
694
+ """The body decoded by its charset (UTF-8 by default), bad bytes replaced.
695
+
696
+ A UTF-8 body drops a leading byte order mark, which JSON parsing refuses.
697
+ """
698
+ charset = self.charset or "utf-8"
699
+ try:
700
+ if codecs.lookup(charset).name == "utf-8":
701
+ charset = "utf-8-sig"
702
+ return self.body.decode(charset, errors="replace")
703
+ except LookupError:
704
+ return self.body.decode("utf-8-sig", errors="replace")
705
+
706
+ @cached_property
707
+ def json(self) -> tuple[bool, Any]:
708
+ """``(parsed, value)``: the body as JSON, when it is JSON."""
709
+ if not self.body.strip():
710
+ return False, None
711
+ try:
712
+ return True, json.loads(self.text)
713
+ except (ValueError, RecursionError):
714
+ return False, None
715
+
716
+
717
+ class _BodyError(Exception):
718
+ """The response body cannot be used: too large, or not decodable."""
719
+
720
+
721
+ class _ResponseTooLarge(_BodyError):
722
+ """The response body exceeds the step's byte budget."""
723
+
724
+
725
+ def _elapsed_ms(started: float) -> int:
726
+ """Milliseconds since ``started`` (a ``time.perf_counter()`` value), rounded."""
727
+ return round((time.perf_counter() - started) * 1000)
728
+
729
+
730
+ class _BoundedDecoder:
731
+ """Decompress a gzip or deflate body without ever producing more than the budget.
732
+
733
+ The step reads the raw body and decodes it here rather than letting httpx
734
+ decode it, because httpx inflates each chunk in full before the size
735
+ check could see it: a small compressed body could expand to gigabytes.
736
+ """
737
+
738
+ def __init__(self, encoding: str, too_large: str) -> None:
739
+ """Start decoding ``encoding``; ``too_large`` is the error once the budget is passed."""
740
+ self._encoding = encoding
741
+ self._too_large = too_large
742
+ self._gzip = encoding in _GZIP_ENCODINGS
743
+ self._raw_deflate = False
744
+ self._started = False
745
+ self._decompressor = zlib.decompressobj(
746
+ zlib.MAX_WBITS | 16 if self._gzip else zlib.MAX_WBITS
747
+ )
748
+
749
+ def decode(self, data: bytes, room: int) -> bytes:
750
+ """The decompressed form of ``data``; raises once more than ``room`` bytes come out.
751
+
752
+ A gzip body may hold several members, one after another: each is
753
+ decoded within the same budget, and zero bytes padding them are skipped.
754
+ """
755
+ out = bytearray()
756
+ pending = data
757
+ while pending:
758
+ if self._decompressor.eof:
759
+ if not self._gzip:
760
+ # Deflate data is a single stream; nothing after its end is read.
761
+ break
762
+ pending = pending.lstrip(b"\0")
763
+ if not pending:
764
+ break
765
+ self._decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16)
766
+ try:
767
+ chunk = self._decompressor.decompress(pending, room - len(out) + 1)
768
+ except zlib.error:
769
+ if self._encoding == "deflate" and not self._started and not self._raw_deflate:
770
+ # Some servers send raw deflate without the zlib wrapper.
771
+ self._raw_deflate = True
772
+ self._decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
773
+ continue
774
+ raise _BodyError(f"the response body is not valid {self._encoding} data") from None
775
+ self._started = True
776
+ out.extend(chunk)
777
+ if len(out) > room:
778
+ raise _ResponseTooLarge(self._too_large)
779
+ if self._decompressor.eof:
780
+ pending = self._decompressor.unused_data
781
+ continue
782
+ tail = self._decompressor.unconsumed_tail
783
+ if not chunk and tail == pending:
784
+ break
785
+ pending = tail
786
+ return bytes(out)
787
+
788
+ def finish(self) -> None:
789
+ """Refuse a body that ended inside its compressed data, cut short on the way."""
790
+ if self._started and not self._decompressor.eof:
791
+ raise _BodyError(
792
+ f"the response body ended before its {self._encoding} data was complete"
793
+ )
794
+
795
+
796
+ async def _read_body(response: httpx.Response, remaining: int, limit_text: str) -> bytes:
797
+ """The response body, decoded, and refused once it passes ``remaining`` bytes.
798
+
799
+ The raw body is read in chunks and decoded here (identity, gzip or
800
+ deflate), so neither the raw nor the decoded body can outgrow the budget.
801
+ A declared Content-Length over the budget is refused before reading,
802
+ unless the response has no body by definition (HEAD, 204, 304) or is
803
+ compressed (the declared length is not the decoded one). A compressed
804
+ body that ends before its compressed data does is refused too.
805
+ """
806
+ too_large = (
807
+ f"the response body is larger than {limit_text}; raise Max Response Size or request less"
808
+ )
809
+ declared = response.headers.get("content-length", "")
810
+ encoding = response.headers.get("content-encoding", "identity").strip().lower()
811
+ has_body = response.request.method != "HEAD" and response.status_code not in {204, 304}
812
+ identity = encoding in _IDENTITY_ENCODINGS
813
+ if has_body and identity and declared.isdigit() and int(declared) > remaining:
814
+ raise _ResponseTooLarge(
815
+ f"the response body ({declared} bytes) is larger than {limit_text}; raise "
816
+ "Max Response Size or request less"
817
+ )
818
+ if response.is_stream_consumed:
819
+ # Only an in-process transport hands back a body already read (and
820
+ # decoded by httpx); a network response is always still a stream.
821
+ content = response.content
822
+ if len(content) > remaining:
823
+ raise _ResponseTooLarge(too_large)
824
+ return content
825
+ decoder: _BoundedDecoder | None = None
826
+ if not identity and (encoding in _GZIP_ENCODINGS or encoding == "deflate"):
827
+ decoder = _BoundedDecoder(encoding, too_large)
828
+ body = bytearray()
829
+ raw_total = 0
830
+ async for raw in response.aiter_raw():
831
+ if not raw:
832
+ continue
833
+ if not identity and decoder is None:
834
+ raise _BodyError(
835
+ f"the response uses content encoding {encoding!r}, which this step does not "
836
+ f"decode (it asks for {ACCEPT_ENCODING} or none); remove any Accept-Encoding "
837
+ "header that asks for it"
838
+ )
839
+ raw_total += len(raw)
840
+ if decoder is None:
841
+ body.extend(raw)
842
+ else:
843
+ if raw_total > remaining + _RAW_SLACK_BYTES:
844
+ raise _ResponseTooLarge(too_large)
845
+ body.extend(decoder.decode(raw, remaining - len(body)))
846
+ if len(body) > remaining:
847
+ raise _ResponseTooLarge(too_large)
848
+ if decoder is not None:
849
+ decoder.finish()
850
+ return bytes(body)
851
+
852
+
853
+ def _strip_for_new_origin(
854
+ headers: httpx.Headers, author_headers: dict[str, str], credential_header: str | None
855
+ ) -> None:
856
+ """Drop credentials and author-set headers before a request leaves the origin."""
857
+ drop = {"authorization", "proxy-authorization"}
858
+ if credential_header:
859
+ drop.add(credential_header.lower())
860
+ drop.update(
861
+ name.lower() for name in author_headers if name.lower() not in _CROSS_ORIGIN_SAFE_HEADERS
862
+ )
863
+ for name in drop:
864
+ headers.pop(name, None)
865
+
866
+
867
+ class HttpRequestHandler(BaseHandler):
868
+ """Send one HTTP(S) request (optionally following next-page links) from the worker."""
869
+
870
+ handler_id = HANDLER_ID
871
+ supported_kinds = [StepKind.ACTION]
872
+ display_name = "HTTP Request"
873
+ description = (
874
+ "Send an HTTP(S) request with any method, headers, body and authentication, and "
875
+ "return the response for later steps."
876
+ )
877
+ category = "Actions"
878
+ targeting = HandlerTargeting(roles=False, ips=False)
879
+ # Secret references reach the handler as written (see resolve_secret_text).
880
+ raw_config_keys = frozenset({"auth_secret", "ca_bundle"})
881
+ config_model = HttpRequestConfig
882
+ default_config = {"method": "GET", "expected_status": "200-299"}
883
+
884
+ def __init__(self, *, transport: httpx.AsyncBaseTransport | None = None) -> None:
885
+ """Use ``transport`` instead of the network (tests only)."""
886
+ # Tests route the client through a mock transport; the host passes nothing.
887
+ self._transport = transport
888
+
889
+ def validate_config(self, config: dict[str, Any]) -> list[str]:
890
+ """Validate the step's fields; templated values are checked again once rendered."""
891
+ errors = super().validate_config(config)
892
+ if errors:
893
+ return errors
894
+ return config_errors(HttpRequestConfig.model_validate(config), allow_templates=True)
895
+
896
+ async def execute(self, ctx: HandlerContext) -> HandlerResult:
897
+ """Validate the rendered config, resolve credentials and TLS trust, then send."""
898
+ try:
899
+ config = self.parse_config(ctx.config, HttpRequestConfig)
900
+ except InvalidConfigError as exc:
901
+ return exc.result()
902
+ return (await self.send(ctx.require_services(), config)).result
903
+
904
+ async def send(self, services: HandlerServices, config: HttpRequestConfig) -> HttpResponse:
905
+ """Send the request ``config`` describes, as an ``http.request`` step would.
906
+
907
+ For other handlers that speak HTTP (the NetBox steps): every safeguard
908
+ of this step applies, and the result also carries the whole parsed
909
+ response. ``config`` is checked as rendered, so it may not hold
910
+ templates.
911
+ """
912
+ errors = config_errors(config, allow_templates=False)
913
+ if errors:
914
+ return HttpResponse(InvalidConfigError(HANDLER_ID, errors).result())
915
+ if getattr(services, "enforce_egress", None) is None:
916
+ # Fail closed: the step dials the URL itself, so without the gate
917
+ # nothing would apply the flow's network policy.
918
+ return HttpResponse(
919
+ HandlerResult(
920
+ success=False,
921
+ error=(
922
+ "this platform cannot apply the flow's network policy to the request, "
923
+ "so it was not sent"
924
+ ),
925
+ summary="Config error",
926
+ )
927
+ )
928
+ try:
929
+ credentials = await self._credentials(services, config)
930
+ verify = await self._tls_verify(services, config)
931
+ body = _json_body(config)
932
+ except ValueError as exc:
933
+ return HttpResponse(
934
+ HandlerResult(
935
+ success=False,
936
+ error=f"Invalid {HANDLER_ID} config: {exc}",
937
+ summary="Config error",
938
+ )
939
+ )
940
+ except RuntimeError as exc:
941
+ return HttpResponse(
942
+ HandlerResult(
943
+ success=False,
944
+ error=f"Could not load the organization's trusted CAs: {exc}",
945
+ summary="Trusted CAs unavailable",
946
+ )
947
+ )
948
+ return await self._run(services, config, credentials, verify, body)
949
+
950
+ @staticmethod
951
+ async def _credentials(services: HandlerServices, config: HttpRequestConfig) -> _Credentials:
952
+ """Resolve Authentication into the header it sets. Raises ``ValueError``."""
953
+ header = _credential_header_name(config)
954
+ if header is None:
955
+ return _Credentials()
956
+ secret = await resolve_secret_text(
957
+ services,
958
+ config.auth_secret,
959
+ field="auth_secret",
960
+ source=f"{HANDLER_ID}.auth_secret",
961
+ )
962
+ if len(secret) < MIN_SCRUB_LENGTH:
963
+ raise ValueError(
964
+ f"auth_secret resolved to a value shorter than {MIN_SCRUB_LENGTH} characters, "
965
+ "too short to keep reliably out of the step's output; use a longer secret"
966
+ )
967
+ # A basic-auth password is base64-encoded; any other secret is sent as is.
968
+ if config.auth_type != "basic" and _header_value_error("auth_secret", secret):
969
+ raise ValueError(
970
+ "auth_secret resolved to a value with line breaks, control characters or "
971
+ "non-ASCII characters, which cannot be sent in a header"
972
+ )
973
+ if config.auth_type == "bearer":
974
+ value = f"Bearer {secret}"
975
+ extra: tuple[str, ...] = ()
976
+ elif config.auth_type == "basic":
977
+ token = base64.b64encode(f"{config.auth_username}:{secret}".encode()).decode()
978
+ value = f"Basic {token}"
979
+ extra = (token,)
980
+ else:
981
+ value = f"{config.auth_prefix}{secret}"
982
+ extra = ()
983
+ return _Credentials(header=header, value=value, secrets=(secret, value, *extra))
984
+
985
+ @staticmethod
986
+ async def _tls_verify(services: HandlerServices, config: HttpRequestConfig) -> Any:
987
+ """``verify`` for the client: False, True, or a context trusting the CA bundle.
988
+
989
+ Without a CA bundle the client trusts its default store plus the run
990
+ organization's trusted CAs (just the default store when it has none,
991
+ or on a host without the service). A CA bundle replaces both: only
992
+ its CAs are trusted. Raises ``RuntimeError`` when the host cannot
993
+ load the organization's list for an ``https://`` URL. A plain
994
+ ``http://`` request is still sent then: its first hop needs no
995
+ trust, and an HTTPS hop it is redirected to trusts the default
996
+ store alone, which is stricter, never looser.
997
+ """
998
+ if not config.verify_tls:
999
+ return False
1000
+ if not config.ca_bundle.strip():
1001
+ try:
1002
+ org_pem = await org_trusted_ca_pem(services)
1003
+ except RuntimeError:
1004
+ if httpx.URL(config.url).scheme == "https":
1005
+ raise
1006
+ return True
1007
+ if org_pem is None:
1008
+ return True
1009
+ try:
1010
+ return add_trusted_cas(httpx.create_ssl_context(trust_env=False), org_pem)
1011
+ except (ssl.SSLError, ValueError) as exc:
1012
+ raise RuntimeError(f"a trusted CA could not be loaded: {exc}") from None
1013
+ pem = await resolve_secret_text(
1014
+ services, config.ca_bundle, field="ca_bundle", source=f"{HANDLER_ID}.ca_bundle"
1015
+ )
1016
+ if _PEM_CERTIFICATE not in pem:
1017
+ raise ValueError("ca_bundle holds no PEM certificate (-----BEGIN CERTIFICATE-----)")
1018
+ try:
1019
+ return ssl.create_default_context(cadata=pem)
1020
+ except (ssl.SSLError, ValueError) as exc:
1021
+ raise ValueError(f"ca_bundle could not be loaded: {exc}") from None
1022
+
1023
+ async def _run(
1024
+ self,
1025
+ services: HandlerServices,
1026
+ config: HttpRequestConfig,
1027
+ credentials: _Credentials,
1028
+ verify: Any,
1029
+ json_body: tuple[bool, Any],
1030
+ ) -> HttpResponse:
1031
+ """Send the request (and next pages), then build the output, evidence and result."""
1032
+ scrub = Scrubber(credentials.secrets)
1033
+ ranges = parse_status_spec(config.expected_status)
1034
+ paginating = config.max_pages > 1
1035
+ next_path = parse_path(config.paginate_next) if paginating else ()
1036
+ items_path = parse_path(config.paginate_items) if paginating else ()
1037
+ limit_text = f"Max Response Size ({config.max_response_bytes} bytes)"
1038
+ if paginating:
1039
+ limit_text += " summed over the pages"
1040
+
1041
+ headers: dict[str, str] = dict(config.headers)
1042
+ lowered = {name.lower() for name in headers}
1043
+ content: bytes | None = None
1044
+ has_json, json_value = json_body
1045
+ if has_json:
1046
+ content = json.dumps(
1047
+ json_value, ensure_ascii=False, separators=(",", ":"), allow_nan=False
1048
+ ).encode("utf-8")
1049
+ if "content-type" not in lowered:
1050
+ headers["Content-Type"] = "application/json"
1051
+ elif config.text_body:
1052
+ content = config.text_body.encode("utf-8")
1053
+ if "content-type" not in lowered:
1054
+ headers["Content-Type"] = "text/plain; charset=utf-8"
1055
+ if credentials.header:
1056
+ headers[credentials.header] = credentials.value
1057
+
1058
+ url = httpx.URL(config.url.strip())
1059
+ if config.query:
1060
+ url = url.copy_merge_params(config.query)
1061
+ display_url = str(url.copy_with(query=None, fragment=None))
1062
+
1063
+ enforce = getattr(services, "enforce_egress", None)
1064
+ exchanges: list[_Exchange] = []
1065
+ pages: list[_Page] = []
1066
+
1067
+ async def gate(request: httpx.Request) -> None:
1068
+ """Every outgoing request, redirect hops and pages included, before it dials.
1069
+
1070
+ The host is checked in the ASCII form the connection dials (see
1071
+ ``url_host``). The egress policy's denial propagates as raised (the
1072
+ step reports the policy's reason); the step's own refusals are
1073
+ RequestRefused.
1074
+ """
1075
+ target = request.url
1076
+ if target.scheme not in SCHEME_PORTS:
1077
+ raise RequestRefused(
1078
+ f"refused to send a request to a {target.scheme!r} URL: only http and "
1079
+ "https are supported"
1080
+ )
1081
+ host = url_host(target)
1082
+ port = url_port(target)
1083
+ if enforce is not None:
1084
+ await enforce(host, port, proto="tcp", kind="http request")
1085
+ await check_destination(host, port)
1086
+
1087
+ async def observe(response: httpx.Response) -> None:
1088
+ """Every response, before httpx reads its Location header.
1089
+
1090
+ The status and headers are recorded first, so a redirect that
1091
+ cannot be followed still reports them. Without Follow redirects the
1092
+ response is the result: its Location is kept from httpx, which
1093
+ would otherwise build a next request from it, and fail the step on
1094
+ one that is not a valid URL.
1095
+ """
1096
+ exchange = exchanges[-1]
1097
+ exchange.status = response.status_code
1098
+ exchange.headers = filtered_response_headers(response.headers)
1099
+ if not response.has_redirect_location:
1100
+ return
1101
+ if not config.follow_redirects:
1102
+ del response.headers["location"]
1103
+ return
1104
+ try:
1105
+ httpx.URL(response.headers["location"])
1106
+ except httpx.InvalidURL as exc:
1107
+ raise RequestRefused(
1108
+ f"cannot follow the HTTP {response.status_code} redirect: its Location "
1109
+ f"header is not a valid URL ({exc})"
1110
+ ) from None
1111
+
1112
+ client_options: dict[str, Any] = {
1113
+ "verify": verify,
1114
+ "trust_env": False,
1115
+ "follow_redirects": False,
1116
+ "timeout": httpx.Timeout(config.request_timeout_seconds),
1117
+ "event_hooks": {"request": [gate], "response": [observe]},
1118
+ # Only what _read_body can decode within the size budget.
1119
+ "headers": {"Accept-Encoding": ACCEPT_ENCODING},
1120
+ }
1121
+ if self._transport is not None:
1122
+ client_options["transport"] = self._transport
1123
+
1124
+ sent_header_names: list[str] = []
1125
+ combined: Any = None
1126
+ items: list[Any] = []
1127
+ more_pages = False
1128
+ failure: str | None = None
1129
+ transient = False
1130
+ started = time.perf_counter()
1131
+
1132
+ async with httpx.AsyncClient(**client_options) as client:
1133
+ request = client.build_request(config.method, url, headers=headers, content=content)
1134
+ origin = request.url
1135
+ sent_header_names = sorted({name.lower() for name in request.headers})
1136
+ fetched = {str(origin)}
1137
+ try:
1138
+ while True:
1139
+ used = sum(len(page.body) for page in pages)
1140
+ page = await self._fetch(
1141
+ client,
1142
+ request,
1143
+ config=config,
1144
+ exchanges=exchanges,
1145
+ remaining=config.max_response_bytes - used,
1146
+ limit_text=limit_text,
1147
+ credential_header=credentials.header,
1148
+ has_body=content is not None,
1149
+ stay_at=origin if paginating else None,
1150
+ )
1151
+ pages.append(page)
1152
+ status = page.exchange.status or 0
1153
+ if not status_matches(status, ranges):
1154
+ where = f"page {len(pages)} answered " if len(pages) > 1 else ""
1155
+ failure = (
1156
+ f"{where}HTTP {status} is not in expected_status "
1157
+ f"{config.expected_status}"
1158
+ )
1159
+ break
1160
+ if not paginating:
1161
+ break
1162
+ parsed, data = page.json
1163
+ if not parsed:
1164
+ failure = f"page {len(pages)} is not JSON, so it cannot be paginated"
1165
+ break
1166
+ page_items = get_path(data, items_path)
1167
+ if not isinstance(page_items, list):
1168
+ failure = (
1169
+ f"page {len(pages)} has no list at paginate_items "
1170
+ f"{config.paginate_items!r}"
1171
+ )
1172
+ break
1173
+ items.extend(page_items)
1174
+ if len(pages) == 1:
1175
+ combined = data
1176
+ next_link = get_path(data, next_path)
1177
+ if isinstance(next_link, str):
1178
+ next_link = next_link.strip()
1179
+ if next_link is None or next_link == "":
1180
+ break
1181
+ if not isinstance(next_link, str):
1182
+ failure = (
1183
+ f"page {len(pages)} has a {type(next_link).__name__} at "
1184
+ f"paginate_next {config.paginate_next!r}, not a URL"
1185
+ )
1186
+ break
1187
+ if len(pages) >= config.max_pages:
1188
+ more_pages = True
1189
+ break
1190
+ try:
1191
+ next_url = page.url.join(next_link)
1192
+ except (httpx.InvalidURL, ValueError) as exc:
1193
+ failure = f"page {len(pages)} has an invalid next-page URL: {exc}"
1194
+ break
1195
+ if not same_origin(next_url, origin):
1196
+ failure = (
1197
+ f"refused the next-page URL of page {len(pages)}: it leaves "
1198
+ f"{origin.scheme}://{origin.netloc.decode('ascii')} (next pages must "
1199
+ "keep the first request's scheme, host and port)"
1200
+ )
1201
+ break
1202
+ if str(next_url) in fetched:
1203
+ failure = (
1204
+ f"page {len(pages)} links back to a page already fetched, so "
1205
+ "pagination stopped"
1206
+ )
1207
+ break
1208
+ fetched.add(str(next_url))
1209
+ request = client.build_request("GET", next_url, headers=headers)
1210
+ except (
1211
+ RequestRefused,
1212
+ _BodyError,
1213
+ httpx.HTTPError,
1214
+ httpx.InvalidURL,
1215
+ TimeoutError,
1216
+ ) as exc:
1217
+ failure = _describe_error(exc, config.request_timeout_seconds)
1218
+ transient = _is_transient(exc)
1219
+
1220
+ elapsed_ms = _elapsed_ms(started)
1221
+ last = exchanges[-1] if exchanges else None
1222
+ last_page = pages[-1] if pages and last is not None and pages[-1].exchange is last else None
1223
+ status = last.status if last is not None else None
1224
+ # A response with the expected status is not ok when the step still
1225
+ # failed on it (its body, a redirect, a page).
1226
+ ok = failure is None and status is not None and status_matches(status, ranges)
1227
+ size_bytes = sum(len(page.body) for page in pages)
1228
+
1229
+ if last_page is not None:
1230
+ # Redacted whole, before any preview is cut from it: a credential
1231
+ # crossing the cut would otherwise survive in part.
1232
+ text = scrub.text(last_page.text)
1233
+ parsed, data = last_page.json
1234
+ if not parsed:
1235
+ data = None
1236
+ body_sha256: str | None = hashlib.sha256(last_page.body).hexdigest()
1237
+ else:
1238
+ text, data, body_sha256 = "", None, None
1239
+ if paginating and failure is None and combined is not None:
1240
+ data = replace_at(combined, items_path, items)
1241
+
1242
+ output: dict[str, Any] = {
1243
+ "status": status,
1244
+ "ok": ok,
1245
+ "url": last.url if last is not None else str(url),
1246
+ "method": config.method,
1247
+ "headers": dict(last.headers) if last is not None else {},
1248
+ "elapsed_ms": elapsed_ms,
1249
+ "size_bytes": size_bytes,
1250
+ "body_sha256": body_sha256,
1251
+ "pages": len(pages),
1252
+ "more_pages": more_pages,
1253
+ }
1254
+ meta: dict[str, Any] = {
1255
+ "method": config.method,
1256
+ "url": exchanges[0].url if exchanges else str(url),
1257
+ "status": status,
1258
+ "elapsed_ms": elapsed_ms,
1259
+ "request_headers": sent_header_names,
1260
+ "response_headers": dict(output["headers"]),
1261
+ "requests": [exchange.record() for exchange in exchanges],
1262
+ "pages": len(pages),
1263
+ "size_bytes": size_bytes,
1264
+ "body_sha256": body_sha256,
1265
+ }
1266
+ if not config.sensitive:
1267
+ data_truncated = data is not None and _json_size(data) > MAX_DATA_OUTPUT_BYTES
1268
+ output["body"] = text[:MAX_BODY_OUTPUT_CHARS]
1269
+ output["body_truncated"] = len(text) > MAX_BODY_OUTPUT_CHARS
1270
+ output["data"] = None if data_truncated else data
1271
+ output["data_truncated"] = data_truncated
1272
+ # Extract reads the whole response, however large its data.
1273
+ output["extracted"] = {
1274
+ name: get_path(data, parse_path(path)) for name, path in config.extract.items()
1275
+ }
1276
+ meta["body_preview"] = text[:EVIDENCE_PREVIEW_CHARS]
1277
+ meta["body_truncated"] = len(text) > EVIDENCE_PREVIEW_CHARS
1278
+ if failure:
1279
+ meta["error"] = failure
1280
+
1281
+ timing = f"{len(pages)} pages, {elapsed_ms} ms" if len(pages) > 1 else f"{elapsed_ms} ms"
1282
+ if status is None:
1283
+ summary = f"{config.method} {display_url} failed ({timing})"
1284
+ elif failure:
1285
+ summary = f"{config.method} {display_url} -> {status}, failed ({timing})"
1286
+ else:
1287
+ summary = f"{config.method} {display_url} -> {status} ({timing})"
1288
+
1289
+ metrics: dict[str, Any] = {"elapsed_ms": elapsed_ms, "size_bytes": size_bytes, "ok": ok}
1290
+ if status is not None:
1291
+ metrics["status"] = status
1292
+ evidence = [
1293
+ {"kind": "transport_meta", "name": "http_request", "content_json": scrub.tree(meta)}
1294
+ ]
1295
+ result = HandlerResult(
1296
+ success=failure is None,
1297
+ summary=scrub.text(summary),
1298
+ metrics=metrics,
1299
+ evidence=evidence,
1300
+ error=scrub.text(failure) if failure else None,
1301
+ output=scrub.tree(output),
1302
+ )
1303
+ return HttpResponse(result, None if config.sensitive else scrub.tree(data), transient)
1304
+
1305
+ @staticmethod
1306
+ async def _fetch(
1307
+ client: httpx.AsyncClient,
1308
+ request: httpx.Request,
1309
+ *,
1310
+ config: HttpRequestConfig,
1311
+ exchanges: list[_Exchange],
1312
+ remaining: int,
1313
+ limit_text: str,
1314
+ credential_header: str | None,
1315
+ has_body: bool = False,
1316
+ stay_at: httpx.URL | None = None,
1317
+ ) -> _Page:
1318
+ """Send ``request``, following redirects when configured; the final response, read.
1319
+
1320
+ A redirect that would repeat the request body (``has_body``) to another
1321
+ scheme, host or port is refused. With ``stay_at`` (while paginating), a
1322
+ redirect away from that URL's scheme, host and port is refused, as a
1323
+ next-page link would be.
1324
+ """
1325
+ hops = 0
1326
+ while True:
1327
+ exchange = _Exchange(method=request.method, url=str(request.url))
1328
+ exchanges.append(exchange)
1329
+ started = time.perf_counter()
1330
+ next_request: httpx.Request | None = None
1331
+ try:
1332
+ async with asyncio.timeout(config.request_timeout_seconds):
1333
+ # The client's response hook records the status and headers.
1334
+ response = await client.send(request, stream=True)
1335
+ try:
1336
+ if config.follow_redirects and response.next_request is not None:
1337
+ # The redirect's own body is never read.
1338
+ next_request = response.next_request
1339
+ else:
1340
+ body = await _read_body(response, remaining, limit_text)
1341
+ return _Page(
1342
+ exchange=exchange,
1343
+ url=response.url,
1344
+ body=body,
1345
+ charset=response.charset_encoding,
1346
+ )
1347
+ finally:
1348
+ await response.aclose()
1349
+ finally:
1350
+ exchange.elapsed_ms = _elapsed_ms(started)
1351
+ if next_request is None: # pragma: no cover - every path above returns or sets it
1352
+ raise RequestRefused("the redirect could not be followed")
1353
+ hops += 1
1354
+ if hops > MAX_REDIRECTS:
1355
+ raise RequestRefused(f"stopped after {MAX_REDIRECTS} redirects")
1356
+ if next_request.url.scheme not in SCHEME_PORTS:
1357
+ raise RequestRefused(
1358
+ f"refused to follow a redirect to a {next_request.url.scheme!r} URL: only "
1359
+ "http and https are supported"
1360
+ )
1361
+ if stay_at is not None and not same_origin(next_request.url, stay_at):
1362
+ raise RequestRefused(
1363
+ f"refused a redirect to {next_request.url.scheme}://"
1364
+ f"{next_request.url.netloc.decode('ascii')} while paginating (every page "
1365
+ "must keep the first request's scheme, host and port)"
1366
+ )
1367
+ # httpx keeps the method and body except where the redirect turns it into a GET.
1368
+ has_body = has_body and next_request.method == request.method
1369
+ if not same_origin(next_request.url, request.url):
1370
+ if has_body:
1371
+ raise RequestRefused(
1372
+ f"refused to follow the HTTP {exchange.status} redirect to "
1373
+ f"{next_request.url.scheme}://{next_request.url.netloc.decode('ascii')}: "
1374
+ "it would send the request body to another scheme, host or port; "
1375
+ "send the request to that URL instead"
1376
+ )
1377
+ _strip_for_new_origin(next_request.headers, config.headers, credential_header)
1378
+ request = next_request
1379
+
1380
+
1381
+ def _json_size(value: Any) -> int:
1382
+ """Bytes of ``value`` as compact UTF-8 JSON."""
1383
+ text = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
1384
+ return len(text.encode("utf-8", errors="surrogatepass"))
1385
+
1386
+
1387
+ #: Failures where the request may not have reached the server (see HttpResponse.transient).
1388
+ _TRANSIENT_ERRORS = (
1389
+ TimeoutError,
1390
+ httpx.TimeoutException,
1391
+ httpx.ConnectError,
1392
+ httpx.ReadError,
1393
+ httpx.WriteError,
1394
+ httpx.RemoteProtocolError,
1395
+ )
1396
+
1397
+
1398
+ #: TLS errors that are a dropped connection rather than a handshake the server
1399
+ #: refused: the peer went away, which the next attempt may not meet again.
1400
+ _DROPPED_TLS_ERRORS = (ssl.SSLEOFError, ssl.SSLSyscallError, ssl.SSLZeroReturnError)
1401
+ _DROPPED_TLS_REASONS = frozenset({"UNEXPECTED_EOF_WHILE_READING"})
1402
+
1403
+
1404
+ def _is_transient(exc: BaseException) -> bool:
1405
+ """Whether the request may not have reached the server (see ``HttpResponse.transient``).
1406
+
1407
+ A connection that failed in the TLS handshake (a certificate the step does
1408
+ not trust, a name mismatch, a protocol the server refuses) is not: the
1409
+ next attempt meets the same server. httpx raises that ``ssl.SSLError`` as
1410
+ a ``ConnectError`` from httpcore's, which re-raises it ``from None``, so
1411
+ the error is found through ``__context__`` where ``__cause__`` is unset.
1412
+ A connection the peer dropped during the handshake stays transient.
1413
+ """
1414
+ if not isinstance(exc, _TRANSIENT_ERRORS):
1415
+ return False
1416
+ seen: set[int] = set()
1417
+ cause: BaseException | None = exc
1418
+ while cause is not None and id(cause) not in seen and len(seen) < 8:
1419
+ seen.add(id(cause))
1420
+ if isinstance(cause, ssl.SSLError) and not _is_dropped_tls(cause):
1421
+ return False
1422
+ cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__
1423
+ return True
1424
+
1425
+
1426
+ def _is_dropped_tls(exc: ssl.SSLError) -> bool:
1427
+ """Whether ``exc`` says the peer went away, not that it refused the handshake."""
1428
+ if isinstance(exc, _DROPPED_TLS_ERRORS):
1429
+ return True
1430
+ return getattr(exc, "reason", None) in _DROPPED_TLS_REASONS
1431
+
1432
+
1433
+ def _describe_error(exc: BaseException, timeout_seconds: int) -> str:
1434
+ """A step error for a request that failed before a usable response."""
1435
+ if isinstance(exc, (RequestRefused, _BodyError)):
1436
+ return str(exc)
1437
+ if isinstance(exc, (TimeoutError, httpx.TimeoutException)):
1438
+ return f"the request timed out after {timeout_seconds} s"
1439
+ detail = str(exc).strip()
1440
+ name = type(exc).__name__
1441
+ return f"request failed: {name}: {detail}" if detail else f"request failed: {name}"
1442
+
1443
+
1444
+ __all__ = [
1445
+ "HANDLER_ID",
1446
+ "HttpRequestConfig",
1447
+ "HttpRequestHandler",
1448
+ "HttpResponse",
1449
+ "config_errors",
1450
+ "parse_status_spec",
1451
+ "status_matches",
1452
+ ]