hegemony-steps-http 0.46.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,33 @@
1
+ # SPDX-FileCopyrightText: 2025-2026 Rexonix s.r.o.
2
+ # SPDX-FileContributor: Jakub Trávník <jakub.travnik@gmail.com>
3
+ #
4
+ # SPDX-License-Identifier: AGPL-3.0-or-later
5
+
6
+ """HTTP(S) request step handler for Hegemony (http.request).
7
+
8
+ Sends one HTTP(S) request from the worker, with any method, headers, body
9
+ and authentication, and returns the response as structured step output.
10
+ A wheel of its own so that a deployment can leave out arbitrary outbound
11
+ requests entirely.
12
+
13
+ Registers under the ``hegemony.step_handlers`` entry-point group; the entry
14
+ point name is the claimed handler-id namespace prefix (``http``).
15
+ Handlers reach every platform facility through ``ctx.services``.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from hegemony_step_sdk import BaseHandler, StepHandlerRegistry
21
+
22
+ from .request import HttpRequestHandler
23
+
24
+ ALL_HANDLERS: tuple[type[BaseHandler], ...] = (HttpRequestHandler,)
25
+
26
+
27
+ def register(registry: StepHandlerRegistry) -> None:
28
+ """Entry point: register this wheel's handlers with the host registry."""
29
+ for handler_class in ALL_HANDLERS:
30
+ registry.register_handler_type(handler_class)
31
+
32
+
33
+ __all__ = ["ALL_HANDLERS", "register", *sorted(cls.__name__ for cls in ALL_HANDLERS)]
@@ -0,0 +1,198 @@
1
+ <!--
2
+ SPDX-FileCopyrightText: 2025-2026 Rexonix s.r.o.
3
+ SPDX-FileContributor: Jakub Trávník <jakub.travnik@gmail.com>
4
+
5
+ SPDX-License-Identifier: AGPL-3.0-or-later
6
+ -->
7
+
8
+ # HTTP Request
9
+
10
+ `http.request` sends one HTTP or HTTPS request from the worker and returns the
11
+ response as structured output for later steps: call a REST API, create or
12
+ update a record, fetch a token, or read a paged list. The step has no targets;
13
+ the URL says where the request goes.
14
+
15
+ ## Using It
16
+
17
+ URL is the only required field: an `https://` or `http://` address such as
18
+ `https://api.example.com/v1/items`, which may use templates such as
19
+ `{{ inputs.base_url }}/items` or `{{ steps.NODE_ID.output.data.next }}`.
20
+ Credentials never go in the URL (`user:password@` is refused): use
21
+ Authentication. Method is GET (the default), POST, PUT, PATCH, DELETE, HEAD or
22
+ OPTIONS. Query Parameters (`NAME=value` lines) are added to the URL, replacing
23
+ a parameter of the same name the URL already carries. Headers (`Name=value`
24
+ lines) are sent as written; their values may use templates. Host,
25
+ Content-Length, Transfer-Encoding and Connection are set by the step.
26
+
27
+ A body is either JSON Body or Text Body, never both, and GET, HEAD and OPTIONS
28
+ send none. JSON Body is sent as compact JSON with `Content-Type:
29
+ application/json`; written as text, it is parsed as JSON first, so a template
30
+ that renders JSON works:
31
+
32
+ ```json
33
+ {"name": "{{ inputs.name }}", "tags": ["managed"]}
34
+ ```
35
+
36
+ A value inserted into JSON text must itself be valid inside it (quotes
37
+ escaped). Text Body is sent as written, with `Content-Type: text/plain;
38
+ charset=utf-8`. A Content-Type in Headers replaces either default, for
39
+ example `application/merge-patch+json`.
40
+
41
+ Authentication is None (the default), Bearer token (`Authorization: Bearer`
42
+ and the secret), Basic (Username, which may not contain a colon, and the
43
+ secret as password), or Custom header: Header Name (default `Authorization`)
44
+ carries Value Prefix (for example `Token ` with its trailing space) followed
45
+ by the secret. Secret takes a bare secret reference such as
46
+ `vault://orgs/acme/secrets/api/token`, a `{{ secret('...') }}` template, or
47
+ literal text; any `scheme://...` text is read as a reference. A Header in
48
+ Headers with the same name as the credential header is refused.
49
+ Authentication needs an `https://` URL, so the credential never travels
50
+ unencrypted; Allow credentials over HTTP (off by default) sends it to an
51
+ `http://` URL anyway, for an internal endpoint that offers no HTTPS.
52
+
53
+ Expected Status lists the status codes that count as success, as values and
54
+ ranges such as `200-299,304`; the default is `200-299`. Timeout (sec), default
55
+ 30, bounds each request: connecting, sending and reading the whole response.
56
+ Follow redirects is off by default, so a 3xx response is the result (add it to
57
+ Expected Status to accept it), whatever its `Location` header holds; switched
58
+ on, the step follows up to 10 redirects and reports the final response, and a
59
+ `Location` that is not a valid URL fails the step, reporting the redirect's
60
+ status. A 302 or 303 becomes a GET without a body, as does a 301 answering a
61
+ POST; other redirects, 307 and 308 among them, repeat the method and body, but
62
+ only on the same scheme, host and port: one that would send the body anywhere
63
+ else is refused.
64
+
65
+ Verify TLS certificate is on by default. The server's certificate is checked
66
+ against the public certificate authorities plus the organization's trusted
67
+ CAs, which administrators manage under Settings, Trusted CAs (the list of the
68
+ organization the run belongs to). CA Bundle trusts only the PEM certificates
69
+ you give for this request instead of both, which pins them; like Secret it
70
+ may be PEM text, a secret reference or a template. Turn verification off only
71
+ for test endpoints.
72
+
73
+ Max Response Size (default 1 MiB, at most 16 MiB) is the largest response
74
+ body the step reads, summed over all pages. A larger body fails the step; it
75
+ is never cut short silently. The step asks for uncompressed, gzip or deflate
76
+ bodies and decompresses them itself, stopping at Max Response Size, so a
77
+ small compressed body cannot expand past it; a gzip body may hold several
78
+ members. A compressed body that ends before its compressed data does, or a
79
+ body in any other encoding, fails the step.
80
+
81
+ ### Extracting values
82
+
83
+ Extract picks values out of the JSON response, one `NAME=path` line each, for
84
+ example `id=results[0].id` or `token=$.auth.token`. A path is keys separated
85
+ by dots, `[n]` for a list item (`[-1]` is the last), `["a.b"]` for a key with
86
+ dots or spaces, and an optional leading `$.`. A path that matches nothing gives
87
+ `null` rather than an error. Later steps read
88
+ `{{ steps.NODE_ID.output.extracted.id }}`.
89
+
90
+ ### Pagination
91
+
92
+ For an API that returns one page at a time with a link to the next, set Next
93
+ Page Path to where that link sits (for example `next`), Items Path to the list
94
+ of items (for example `results`), and Max Pages above 1 (at most 100). The
95
+ step then follows the links with GET, the same headers and credentials, until
96
+ the link is missing, `null` or empty, or Max Pages is reached; `more_pages`
97
+ tells which. A relative link is resolved against the page's URL. Next pages,
98
+ and any redirect while paginating, must stay on the first request's scheme,
99
+ host and port, and a link back to a page already fetched stops pagination
100
+ with an error. Pagination needs the GET method, and each page must be JSON
101
+ with a list at Items Path.
102
+
103
+ ## Output
104
+
105
+ The summary reads like `GET https://api.example.com/v1/items -> 200 (85 ms)`
106
+ (the query string left out), and `(3 pages, 240 ms)` when paginating. A step
107
+ that failed reads `-> 404, failed (85 ms)` when a response arrived, and
108
+ `failed (85 ms)` when none did. Later steps read the output as
109
+ `steps.NODE_ID.output.KEY`:
110
+
111
+ - `status`: the HTTP status of the last response (`null` if none arrived).
112
+ - `ok`: whether the step succeeded: that status is in Expected Status, and
113
+ nothing else failed (the body, a redirect or a page).
114
+ - `url`: the URL of the last request, after redirects; `method`: the method.
115
+ - `headers`: the last response's headers, names lower-cased; repeated headers
116
+ joined with `, `. `set-cookie`, `set-cookie2`, `authorization`,
117
+ `proxy-authorization` and `cookie` are always left out.
118
+ - `body`: the last response's body as text, at most 64 KiB, with
119
+ `body_truncated` telling whether it was cut.
120
+ - `data`: the body parsed as JSON (a leading UTF-8 byte order mark is
121
+ ignored), or `null` when it is not JSON. With pagination, the first page
122
+ with the list at Items Path replaced by the items of every page. Step
123
+ outputs stay small, so data over 64 KiB as compact JSON is left out:
124
+ `data` is then `null` and `data_truncated` true (it is false otherwise).
125
+ - `extracted`: the Extract values by name, read from the whole response even
126
+ when `data` is left out.
127
+ - `elapsed_ms`: the time spent on all requests; `size_bytes`: the body bytes
128
+ read, all pages together; `body_sha256`: the SHA-256 of the last body.
129
+ - `pages`: the number of pages fetched; `more_pages`: whether Max Pages
130
+ stopped the step while another page was linked.
131
+
132
+ When the status is not expected, the step fails but its output is still
133
+ filled in from that response, so a failure branch can read `status`, `body`
134
+ and `data`. Metrics record `elapsed_ms`, `size_bytes`, `ok` and `status`. One
135
+ `transport_meta` evidence record named `http_request` keeps the method, the
136
+ URL as sent, every request made (redirect hops and pages, each with its status
137
+ and time), the names of the request headers sent, the response headers as
138
+ above, the first 4096 characters of the body, its size and SHA-256.
139
+
140
+ Sensitive response keeps `body`, `body_truncated`, `data`, `data_truncated`
141
+ and `extracted` out of both output and evidence, for a response that carries
142
+ a secret; the status, headers, size and SHA-256 remain. Extract cannot be
143
+ combined with it.
144
+
145
+ ## Security
146
+
147
+ - **Destinations.** Only `http` and `https` are sent; a redirect to any other
148
+ scheme is refused. Before any connection, every request, redirect hops and
149
+ next pages included, passes the flow's egress policy, and a request to a
150
+ loopback, link-local, unspecified or multicast address is refused: a literal
151
+ IP is judged as written, a host name by every address it resolves to (it is
152
+ refused if any of them is such an address). An internationalized name is
153
+ checked, and passed to the egress policy, in the ASCII (`xn--`) form the
154
+ connection uses. The cloud instance metadata services at `169.254.169.254`,
155
+ `fd00:ec2::254` and `100.100.100.200` are refused too. Private network
156
+ addresses are left to the egress policy.
157
+ - **Worker settings.** The step ignores the worker's proxy variables, `.netrc`
158
+ and certificate-file variables: the request goes directly to the URL, and
159
+ trusts the public authorities plus the organization's trusted CAs, or CA
160
+ Bundle only.
161
+ - **Credentials.** Credentials are sent only to the URL's own scheme, host and
162
+ port: a redirect to anywhere else, even to the same host over the other
163
+ scheme, drops them, and drops every header set in Headers except Accept,
164
+ Accept-Language, Content-Type and User-Agent. So put tokens in
165
+ Authentication, not in Headers. A redirect that would repeat the request
166
+ body elsewhere is refused.
167
+ - **What is recorded.** The secret, request header values and the request body
168
+ never appear in the output, evidence, summary or error; request headers are
169
+ recorded by name only. A secret the response repeats, as is, escaped in
170
+ a JSON string or as a JSON number, is replaced with `[redacted]` in the
171
+ output and evidence; the body is redacted whole before Body and the
172
+ evidence preview are cut from it.
173
+ - **Redaction.** A value the platform resolves from a secret reference or
174
+ template is redacted from the run's logs. Literal text in Secret is not
175
+ registered for redaction and is stored in the flow as written, so prefer a
176
+ secret reference.
177
+
178
+ ## When It Fails
179
+
180
+ The step fails before sending anything when the config is invalid: a URL that
181
+ is not `http(s)` or carries credentials, both bodies or a body on GET, HEAD or
182
+ OPTIONS, JSON Body text that is not valid JSON once rendered, an Expected
183
+ Status that does not parse, an invalid path, a missing Secret or Username,
184
+ Authentication on an `http://` URL without Allow credentials over HTTP, or
185
+ incomplete pagination settings. It also fails when a secret reference
186
+ resolves to nothing, the secret is shorter than 4 characters (too short to
187
+ keep reliably out of the output), CA Bundle holds no certificate, or the
188
+ platform cannot apply the flow's network policy to the request.
189
+
190
+ During the request it fails on a refused destination, a DNS failure or a
191
+ host name that is not valid, a connection or TLS error, a timeout, more than
192
+ 10 redirects, a redirect whose `Location` is not a valid URL or that would
193
+ send the request body to another origin, a response over Max Response Size,
194
+ in an encoding the step does not decode or cut short inside its compressed
195
+ data, a status outside Expected Status (output still filled in), or a page
196
+ that is not JSON, has no list at Items Path, links to another origin or back
197
+ to an earlier page, or redirects to another origin. A denial by the flow's
198
+ egress policy fails the step with the policy's own reason.
@@ -0,0 +1,121 @@
1
+ # SPDX-FileCopyrightText: 2025-2026 Rexonix s.r.o.
2
+ # SPDX-FileContributor: Jakub Trávník <jakub.travnik@gmail.com>
3
+ #
4
+ # SPDX-License-Identifier: AGPL-3.0-or-later
5
+
6
+ """A small path language into parsed JSON: ``a.b[0].c``, optionally ``$.``-prefixed.
7
+
8
+ Deliberately not JSONPath: no wildcards, filters or recursion. A path names
9
+ one value. Segments are dot-separated keys, ``[n]`` list indexes (negative
10
+ counts from the end), and ``["key"]`` / ``['key']`` for keys holding dots,
11
+ brackets, quotes or spaces. ``$`` alone (or an empty path after ``$``) names
12
+ the whole document.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import re
18
+ from typing import Any
19
+
20
+ #: One path segment after the first: ``.key``, ``[3]``, ``[-1]``, ``["a.b"]``.
21
+ _SEGMENT_RE = re.compile(
22
+ r"""\.(?P<key>[^.\[\]'"\s]+)"""
23
+ # At most 18 digits: Python refuses to read a 4,301-digit int (ValueError,
24
+ # not PathError), and no list is longer than 10**18 anyway.
25
+ r"""|\[(?P<index>-?\d{1,18})\]"""
26
+ r"""|\[(?P<quote>["'])(?P<quoted>.*?)(?P=quote)\]"""
27
+ )
28
+
29
+ PathPart = str | int
30
+
31
+
32
+ class PathError(ValueError):
33
+ """A path that does not parse."""
34
+
35
+
36
+ def parse_path(path: str) -> tuple[PathPart, ...]:
37
+ """The keys and indexes ``path`` names, outermost first.
38
+
39
+ Raises :class:`PathError` for a malformed path.
40
+ """
41
+ if not isinstance(path, str):
42
+ raise PathError("a path must be a string")
43
+ text = path.strip()
44
+ if not text:
45
+ raise PathError("the path is empty")
46
+ if text.startswith("$"):
47
+ text = text[1:]
48
+ elif not text.startswith((".", "[")):
49
+ text = "." + text
50
+ parts: list[PathPart] = []
51
+ position = 0
52
+ while position < len(text):
53
+ match = _SEGMENT_RE.match(text, position)
54
+ if match is None:
55
+ raise PathError(
56
+ f"path {path!r} is not valid at {text[position:]!r}: use keys separated by "
57
+ 'dots, [n] for list items, and ["key"] for keys with special characters'
58
+ )
59
+ if match.group("key") is not None:
60
+ parts.append(match.group("key"))
61
+ elif match.group("index") is not None:
62
+ parts.append(int(match.group("index")))
63
+ else:
64
+ parts.append(match.group("quoted"))
65
+ position = match.end()
66
+ return tuple(parts)
67
+
68
+
69
+ def _step(current: Any, part: PathPart) -> tuple[bool, Any]:
70
+ """``(found, value)`` one segment below ``current``."""
71
+ if isinstance(current, dict):
72
+ if isinstance(part, str) and part in current:
73
+ return True, current[part]
74
+ return False, None
75
+ if isinstance(current, list):
76
+ index: int | None = None
77
+ if isinstance(part, int):
78
+ index = part
79
+ elif re.fullmatch(r"-?\d{1,18}", part):
80
+ index = int(part)
81
+ if index is not None and -len(current) <= index < len(current):
82
+ return True, current[index]
83
+ return False, None
84
+
85
+
86
+ def get_path(data: Any, parts: tuple[PathPart, ...]) -> Any:
87
+ """The value at ``parts`` in ``data``, or None when any segment is missing."""
88
+ current = data
89
+ for part in parts:
90
+ found, current = _step(current, part)
91
+ if not found:
92
+ return None
93
+ return current
94
+
95
+
96
+ def replace_at(data: Any, parts: tuple[PathPart, ...], value: Any) -> Any:
97
+ """``data`` with the value at ``parts`` replaced by ``value`` (in place); returns the root.
98
+
99
+ The path must exist (raises :class:`PathError` otherwise); an empty path
100
+ replaces the whole document.
101
+ """
102
+ if not parts:
103
+ return value
104
+ parent = data
105
+ for part in parts[:-1]:
106
+ found, parent = _step(parent, part)
107
+ if not found:
108
+ raise PathError("the path does not exist in the document")
109
+ last = parts[-1]
110
+ if isinstance(parent, dict) and isinstance(last, str) and last in parent:
111
+ parent[last] = value
112
+ return data
113
+ if isinstance(parent, list):
114
+ found, _ = _step(parent, last)
115
+ if found:
116
+ parent[int(last)] = value
117
+ return data
118
+ raise PathError("the path does not exist in the document")
119
+
120
+
121
+ __all__ = ["PathError", "get_path", "parse_path", "replace_at"]