hegemony-steps-http 0.46.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hegemony_steps_http/__init__.py +33 -0
- hegemony_steps_http/docs/http.request.md +198 -0
- hegemony_steps_http/jsonpath.py +121 -0
- hegemony_steps_http/request.py +1452 -0
- hegemony_steps_http/security.py +314 -0
- hegemony_steps_http-0.46.1.dist-info/METADATA +40 -0
- hegemony_steps_http-0.46.1.dist-info/RECORD +10 -0
- hegemony_steps_http-0.46.1.dist-info/WHEEL +4 -0
- hegemony_steps_http-0.46.1.dist-info/entry_points.txt +3 -0
- hegemony_steps_http-0.46.1.dist-info/licenses/LICENSE-AGPL-3.0-or-later.txt +235 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2025-2026 Rexonix s.r.o.
|
|
2
|
+
# SPDX-FileContributor: Jakub Trávník <jakub.travnik@gmail.com>
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
5
|
+
|
|
6
|
+
"""HTTP(S) request step handler for Hegemony (http.request).
|
|
7
|
+
|
|
8
|
+
Sends one HTTP(S) request from the worker, with any method, headers, body
|
|
9
|
+
and authentication, and returns the response as structured step output.
|
|
10
|
+
A wheel of its own so that a deployment can leave out arbitrary outbound
|
|
11
|
+
requests entirely.
|
|
12
|
+
|
|
13
|
+
Registers under the ``hegemony.step_handlers`` entry-point group; the entry
|
|
14
|
+
point name is the claimed handler-id namespace prefix (``http``).
|
|
15
|
+
Handlers reach every platform facility through ``ctx.services``.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from hegemony_step_sdk import BaseHandler, StepHandlerRegistry
|
|
21
|
+
|
|
22
|
+
from .request import HttpRequestHandler
|
|
23
|
+
|
|
24
|
+
ALL_HANDLERS: tuple[type[BaseHandler], ...] = (HttpRequestHandler,)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def register(registry: StepHandlerRegistry) -> None:
|
|
28
|
+
"""Entry point: register this wheel's handlers with the host registry."""
|
|
29
|
+
for handler_class in ALL_HANDLERS:
|
|
30
|
+
registry.register_handler_type(handler_class)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
__all__ = ["ALL_HANDLERS", "register", *sorted(cls.__name__ for cls in ALL_HANDLERS)]
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
SPDX-FileCopyrightText: 2025-2026 Rexonix s.r.o.
|
|
3
|
+
SPDX-FileContributor: Jakub Trávník <jakub.travnik@gmail.com>
|
|
4
|
+
|
|
5
|
+
SPDX-License-Identifier: AGPL-3.0-or-later
|
|
6
|
+
-->
|
|
7
|
+
|
|
8
|
+
# HTTP Request
|
|
9
|
+
|
|
10
|
+
`http.request` sends one HTTP or HTTPS request from the worker and returns the
|
|
11
|
+
response as structured output for later steps: call a REST API, create or
|
|
12
|
+
update a record, fetch a token, or read a paged list. The step has no targets;
|
|
13
|
+
the URL says where the request goes.
|
|
14
|
+
|
|
15
|
+
## Using It
|
|
16
|
+
|
|
17
|
+
URL is the only required field: an `https://` or `http://` address such as
|
|
18
|
+
`https://api.example.com/v1/items`, which may use templates such as
|
|
19
|
+
`{{ inputs.base_url }}/items` or `{{ steps.NODE_ID.output.data.next }}`.
|
|
20
|
+
Credentials never go in the URL (`user:password@` is refused): use
|
|
21
|
+
Authentication. Method is GET (the default), POST, PUT, PATCH, DELETE, HEAD or
|
|
22
|
+
OPTIONS. Query Parameters (`NAME=value` lines) are added to the URL, replacing
|
|
23
|
+
a parameter of the same name the URL already carries. Headers (`Name=value`
|
|
24
|
+
lines) are sent as written; their values may use templates. Host,
|
|
25
|
+
Content-Length, Transfer-Encoding and Connection are set by the step.
|
|
26
|
+
|
|
27
|
+
A body is either JSON Body or Text Body, never both, and GET, HEAD and OPTIONS
|
|
28
|
+
send none. JSON Body is sent as compact JSON with `Content-Type:
|
|
29
|
+
application/json`; written as text, it is parsed as JSON first, so a template
|
|
30
|
+
that renders JSON works:
|
|
31
|
+
|
|
32
|
+
```json
|
|
33
|
+
{"name": "{{ inputs.name }}", "tags": ["managed"]}
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
A value inserted into JSON text must itself be valid inside it (quotes
|
|
37
|
+
escaped). Text Body is sent as written, with `Content-Type: text/plain;
|
|
38
|
+
charset=utf-8`. A Content-Type in Headers replaces either default, for
|
|
39
|
+
example `application/merge-patch+json`.
|
|
40
|
+
|
|
41
|
+
Authentication is None (the default), Bearer token (`Authorization: Bearer`
|
|
42
|
+
and the secret), Basic (Username, which may not contain a colon, and the
|
|
43
|
+
secret as password), or Custom header: Header Name (default `Authorization`)
|
|
44
|
+
carries Value Prefix (for example `Token ` with its trailing space) followed
|
|
45
|
+
by the secret. Secret takes a bare secret reference such as
|
|
46
|
+
`vault://orgs/acme/secrets/api/token`, a `{{ secret('...') }}` template, or
|
|
47
|
+
literal text; any `scheme://...` text is read as a reference. A Header in
|
|
48
|
+
Headers with the same name as the credential header is refused.
|
|
49
|
+
Authentication needs an `https://` URL, so the credential never travels
|
|
50
|
+
unencrypted; Allow credentials over HTTP (off by default) sends it to an
|
|
51
|
+
`http://` URL anyway, for an internal endpoint that offers no HTTPS.
|
|
52
|
+
|
|
53
|
+
Expected Status lists the status codes that count as success, as values and
|
|
54
|
+
ranges such as `200-299,304`; the default is `200-299`. Timeout (sec), default
|
|
55
|
+
30, bounds each request: connecting, sending and reading the whole response.
|
|
56
|
+
Follow redirects is off by default, so a 3xx response is the result (add it to
|
|
57
|
+
Expected Status to accept it), whatever its `Location` header holds; switched
|
|
58
|
+
on, the step follows up to 10 redirects and reports the final response, and a
|
|
59
|
+
`Location` that is not a valid URL fails the step, reporting the redirect's
|
|
60
|
+
status. A 302 or 303 becomes a GET without a body, as does a 301 answering a
|
|
61
|
+
POST; other redirects, 307 and 308 among them, repeat the method and body, but
|
|
62
|
+
only on the same scheme, host and port: one that would send the body anywhere
|
|
63
|
+
else is refused.
|
|
64
|
+
|
|
65
|
+
Verify TLS certificate is on by default. The server's certificate is checked
|
|
66
|
+
against the public certificate authorities plus the organization's trusted
|
|
67
|
+
CAs, which administrators manage under Settings, Trusted CAs (the list of the
|
|
68
|
+
organization the run belongs to). CA Bundle trusts only the PEM certificates
|
|
69
|
+
you give for this request instead of both, which pins them; like Secret it
|
|
70
|
+
may be PEM text, a secret reference or a template. Turn verification off only
|
|
71
|
+
for test endpoints.
|
|
72
|
+
|
|
73
|
+
Max Response Size (default 1 MiB, at most 16 MiB) is the largest response
|
|
74
|
+
body the step reads, summed over all pages. A larger body fails the step; it
|
|
75
|
+
is never cut short silently. The step asks for uncompressed, gzip or deflate
|
|
76
|
+
bodies and decompresses them itself, stopping at Max Response Size, so a
|
|
77
|
+
small compressed body cannot expand past it; a gzip body may hold several
|
|
78
|
+
members. A compressed body that ends before its compressed data does, or a
|
|
79
|
+
body in any other encoding, fails the step.
|
|
80
|
+
|
|
81
|
+
### Extracting values
|
|
82
|
+
|
|
83
|
+
Extract picks values out of the JSON response, one `NAME=path` line each, for
|
|
84
|
+
example `id=results[0].id` or `token=$.auth.token`. A path is keys separated
|
|
85
|
+
by dots, `[n]` for a list item (`[-1]` is the last), `["a.b"]` for a key with
|
|
86
|
+
dots or spaces, and an optional leading `$.`. A path that matches nothing gives
|
|
87
|
+
`null` rather than an error. Later steps read
|
|
88
|
+
`{{ steps.NODE_ID.output.extracted.id }}`.
|
|
89
|
+
|
|
90
|
+
### Pagination
|
|
91
|
+
|
|
92
|
+
For an API that returns one page at a time with a link to the next, set Next
|
|
93
|
+
Page Path to where that link sits (for example `next`), Items Path to the list
|
|
94
|
+
of items (for example `results`), and Max Pages above 1 (at most 100). The
|
|
95
|
+
step then follows the links with GET, the same headers and credentials, until
|
|
96
|
+
the link is missing, `null` or empty, or Max Pages is reached; `more_pages`
|
|
97
|
+
tells which. A relative link is resolved against the page's URL. Next pages,
|
|
98
|
+
and any redirect while paginating, must stay on the first request's scheme,
|
|
99
|
+
host and port, and a link back to a page already fetched stops pagination
|
|
100
|
+
with an error. Pagination needs the GET method, and each page must be JSON
|
|
101
|
+
with a list at Items Path.
|
|
102
|
+
|
|
103
|
+
## Output
|
|
104
|
+
|
|
105
|
+
The summary reads like `GET https://api.example.com/v1/items -> 200 (85 ms)`
|
|
106
|
+
(the query string left out), and `(3 pages, 240 ms)` when paginating. A step
|
|
107
|
+
that failed reads `-> 404, failed (85 ms)` when a response arrived, and
|
|
108
|
+
`failed (85 ms)` when none did. Later steps read the output as
|
|
109
|
+
`steps.NODE_ID.output.KEY`:
|
|
110
|
+
|
|
111
|
+
- `status`: the HTTP status of the last response (`null` if none arrived).
|
|
112
|
+
- `ok`: whether the step succeeded: that status is in Expected Status, and
|
|
113
|
+
nothing else failed (the body, a redirect or a page).
|
|
114
|
+
- `url`: the URL of the last request, after redirects; `method`: the method.
|
|
115
|
+
- `headers`: the last response's headers, names lower-cased; repeated headers
|
|
116
|
+
joined with `, `. `set-cookie`, `set-cookie2`, `authorization`,
|
|
117
|
+
`proxy-authorization` and `cookie` are always left out.
|
|
118
|
+
- `body`: the last response's body as text, at most 64 KiB, with
|
|
119
|
+
`body_truncated` telling whether it was cut.
|
|
120
|
+
- `data`: the body parsed as JSON (a leading UTF-8 byte order mark is
|
|
121
|
+
ignored), or `null` when it is not JSON. With pagination, the first page
|
|
122
|
+
with the list at Items Path replaced by the items of every page. Step
|
|
123
|
+
outputs stay small, so data over 64 KiB as compact JSON is left out:
|
|
124
|
+
`data` is then `null` and `data_truncated` true (it is false otherwise).
|
|
125
|
+
- `extracted`: the Extract values by name, read from the whole response even
|
|
126
|
+
when `data` is left out.
|
|
127
|
+
- `elapsed_ms`: the time spent on all requests; `size_bytes`: the body bytes
|
|
128
|
+
read, all pages together; `body_sha256`: the SHA-256 of the last body.
|
|
129
|
+
- `pages`: the number of pages fetched; `more_pages`: whether Max Pages
|
|
130
|
+
stopped the step while another page was linked.
|
|
131
|
+
|
|
132
|
+
When the status is not expected, the step fails but its output is still
|
|
133
|
+
filled in from that response, so a failure branch can read `status`, `body`
|
|
134
|
+
and `data`. Metrics record `elapsed_ms`, `size_bytes`, `ok` and `status`. One
|
|
135
|
+
`transport_meta` evidence record named `http_request` keeps the method, the
|
|
136
|
+
URL as sent, every request made (redirect hops and pages, each with its status
|
|
137
|
+
and time), the names of the request headers sent, the response headers as
|
|
138
|
+
above, the first 4096 characters of the body, its size and SHA-256.
|
|
139
|
+
|
|
140
|
+
Sensitive response keeps `body`, `body_truncated`, `data`, `data_truncated`
|
|
141
|
+
and `extracted` out of both output and evidence, for a response that carries
|
|
142
|
+
a secret; the status, headers, size and SHA-256 remain. Extract cannot be
|
|
143
|
+
combined with it.
|
|
144
|
+
|
|
145
|
+
## Security
|
|
146
|
+
|
|
147
|
+
- **Destinations.** Only `http` and `https` are sent; a redirect to any other
|
|
148
|
+
scheme is refused. Before any connection, every request, redirect hops and
|
|
149
|
+
next pages included, passes the flow's egress policy, and a request to a
|
|
150
|
+
loopback, link-local, unspecified or multicast address is refused: a literal
|
|
151
|
+
IP is judged as written, a host name by every address it resolves to (it is
|
|
152
|
+
refused if any of them is such an address). An internationalized name is
|
|
153
|
+
checked, and passed to the egress policy, in the ASCII (`xn--`) form the
|
|
154
|
+
connection uses. The cloud instance metadata services at `169.254.169.254`,
|
|
155
|
+
`fd00:ec2::254` and `100.100.100.200` are refused too. Private network
|
|
156
|
+
addresses are left to the egress policy.
|
|
157
|
+
- **Worker settings.** The step ignores the worker's proxy variables, `.netrc`
|
|
158
|
+
and certificate-file variables: the request goes directly to the URL, and
|
|
159
|
+
trusts the public authorities plus the organization's trusted CAs, or CA
|
|
160
|
+
Bundle only.
|
|
161
|
+
- **Credentials.** Credentials are sent only to the URL's own scheme, host and
|
|
162
|
+
port: a redirect to anywhere else, even to the same host over the other
|
|
163
|
+
scheme, drops them, and drops every header set in Headers except Accept,
|
|
164
|
+
Accept-Language, Content-Type and User-Agent. So put tokens in
|
|
165
|
+
Authentication, not in Headers. A redirect that would repeat the request
|
|
166
|
+
body elsewhere is refused.
|
|
167
|
+
- **What is recorded.** The secret, request header values and the request body
|
|
168
|
+
never appear in the output, evidence, summary or error; request headers are
|
|
169
|
+
recorded by name only. A secret the response repeats, as is, escaped in
|
|
170
|
+
a JSON string or as a JSON number, is replaced with `[redacted]` in the
|
|
171
|
+
output and evidence; the body is redacted whole before Body and the
|
|
172
|
+
evidence preview are cut from it.
|
|
173
|
+
- **Redaction.** A value the platform resolves from a secret reference or
|
|
174
|
+
template is redacted from the run's logs. Literal text in Secret is not
|
|
175
|
+
registered for redaction and is stored in the flow as written, so prefer a
|
|
176
|
+
secret reference.
|
|
177
|
+
|
|
178
|
+
## When It Fails
|
|
179
|
+
|
|
180
|
+
The step fails before sending anything when the config is invalid: a URL that
|
|
181
|
+
is not `http(s)` or carries credentials, both bodies or a body on GET, HEAD or
|
|
182
|
+
OPTIONS, JSON Body text that is not valid JSON once rendered, an Expected
|
|
183
|
+
Status that does not parse, an invalid path, a missing Secret or Username,
|
|
184
|
+
Authentication on an `http://` URL without Allow credentials over HTTP, or
|
|
185
|
+
incomplete pagination settings. It also fails when a secret reference
|
|
186
|
+
resolves to nothing, the secret is shorter than 4 characters (too short to
|
|
187
|
+
keep reliably out of the output), CA Bundle holds no certificate, or the
|
|
188
|
+
platform cannot apply the flow's network policy to the request.
|
|
189
|
+
|
|
190
|
+
During the request it fails on a refused destination, a DNS failure or a
|
|
191
|
+
host name that is not valid, a connection or TLS error, a timeout, more than
|
|
192
|
+
10 redirects, a redirect whose `Location` is not a valid URL or that would
|
|
193
|
+
send the request body to another origin, a response over Max Response Size,
|
|
194
|
+
in an encoding the step does not decode or cut short inside its compressed
|
|
195
|
+
data, a status outside Expected Status (output still filled in), or a page
|
|
196
|
+
that is not JSON, has no list at Items Path, links to another origin or back
|
|
197
|
+
to an earlier page, or redirects to another origin. A denial by the flow's
|
|
198
|
+
egress policy fails the step with the policy's own reason.
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2025-2026 Rexonix s.r.o.
|
|
2
|
+
# SPDX-FileContributor: Jakub Trávník <jakub.travnik@gmail.com>
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
5
|
+
|
|
6
|
+
"""A small path language into parsed JSON: ``a.b[0].c``, optionally ``$.``-prefixed.
|
|
7
|
+
|
|
8
|
+
Deliberately not JSONPath: no wildcards, filters or recursion. A path names
|
|
9
|
+
one value. Segments are dot-separated keys, ``[n]`` list indexes (negative
|
|
10
|
+
counts from the end), and ``["key"]`` / ``['key']`` for keys holding dots,
|
|
11
|
+
brackets, quotes or spaces. ``$`` alone (or an empty path after ``$``) names
|
|
12
|
+
the whole document.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
#: One path segment after the first: ``.key``, ``[3]``, ``[-1]``, ``["a.b"]``.
|
|
21
|
+
_SEGMENT_RE = re.compile(
|
|
22
|
+
r"""\.(?P<key>[^.\[\]'"\s]+)"""
|
|
23
|
+
# At most 18 digits: Python refuses to read a 4,301-digit int (ValueError,
|
|
24
|
+
# not PathError), and no list is longer than 10**18 anyway.
|
|
25
|
+
r"""|\[(?P<index>-?\d{1,18})\]"""
|
|
26
|
+
r"""|\[(?P<quote>["'])(?P<quoted>.*?)(?P=quote)\]"""
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
PathPart = str | int
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class PathError(ValueError):
|
|
33
|
+
"""A path that does not parse."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def parse_path(path: str) -> tuple[PathPart, ...]:
|
|
37
|
+
"""The keys and indexes ``path`` names, outermost first.
|
|
38
|
+
|
|
39
|
+
Raises :class:`PathError` for a malformed path.
|
|
40
|
+
"""
|
|
41
|
+
if not isinstance(path, str):
|
|
42
|
+
raise PathError("a path must be a string")
|
|
43
|
+
text = path.strip()
|
|
44
|
+
if not text:
|
|
45
|
+
raise PathError("the path is empty")
|
|
46
|
+
if text.startswith("$"):
|
|
47
|
+
text = text[1:]
|
|
48
|
+
elif not text.startswith((".", "[")):
|
|
49
|
+
text = "." + text
|
|
50
|
+
parts: list[PathPart] = []
|
|
51
|
+
position = 0
|
|
52
|
+
while position < len(text):
|
|
53
|
+
match = _SEGMENT_RE.match(text, position)
|
|
54
|
+
if match is None:
|
|
55
|
+
raise PathError(
|
|
56
|
+
f"path {path!r} is not valid at {text[position:]!r}: use keys separated by "
|
|
57
|
+
'dots, [n] for list items, and ["key"] for keys with special characters'
|
|
58
|
+
)
|
|
59
|
+
if match.group("key") is not None:
|
|
60
|
+
parts.append(match.group("key"))
|
|
61
|
+
elif match.group("index") is not None:
|
|
62
|
+
parts.append(int(match.group("index")))
|
|
63
|
+
else:
|
|
64
|
+
parts.append(match.group("quoted"))
|
|
65
|
+
position = match.end()
|
|
66
|
+
return tuple(parts)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _step(current: Any, part: PathPart) -> tuple[bool, Any]:
|
|
70
|
+
"""``(found, value)`` one segment below ``current``."""
|
|
71
|
+
if isinstance(current, dict):
|
|
72
|
+
if isinstance(part, str) and part in current:
|
|
73
|
+
return True, current[part]
|
|
74
|
+
return False, None
|
|
75
|
+
if isinstance(current, list):
|
|
76
|
+
index: int | None = None
|
|
77
|
+
if isinstance(part, int):
|
|
78
|
+
index = part
|
|
79
|
+
elif re.fullmatch(r"-?\d{1,18}", part):
|
|
80
|
+
index = int(part)
|
|
81
|
+
if index is not None and -len(current) <= index < len(current):
|
|
82
|
+
return True, current[index]
|
|
83
|
+
return False, None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def get_path(data: Any, parts: tuple[PathPart, ...]) -> Any:
|
|
87
|
+
"""The value at ``parts`` in ``data``, or None when any segment is missing."""
|
|
88
|
+
current = data
|
|
89
|
+
for part in parts:
|
|
90
|
+
found, current = _step(current, part)
|
|
91
|
+
if not found:
|
|
92
|
+
return None
|
|
93
|
+
return current
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def replace_at(data: Any, parts: tuple[PathPart, ...], value: Any) -> Any:
|
|
97
|
+
"""``data`` with the value at ``parts`` replaced by ``value`` (in place); returns the root.
|
|
98
|
+
|
|
99
|
+
The path must exist (raises :class:`PathError` otherwise); an empty path
|
|
100
|
+
replaces the whole document.
|
|
101
|
+
"""
|
|
102
|
+
if not parts:
|
|
103
|
+
return value
|
|
104
|
+
parent = data
|
|
105
|
+
for part in parts[:-1]:
|
|
106
|
+
found, parent = _step(parent, part)
|
|
107
|
+
if not found:
|
|
108
|
+
raise PathError("the path does not exist in the document")
|
|
109
|
+
last = parts[-1]
|
|
110
|
+
if isinstance(parent, dict) and isinstance(last, str) and last in parent:
|
|
111
|
+
parent[last] = value
|
|
112
|
+
return data
|
|
113
|
+
if isinstance(parent, list):
|
|
114
|
+
found, _ = _step(parent, last)
|
|
115
|
+
if found:
|
|
116
|
+
parent[int(last)] = value
|
|
117
|
+
return data
|
|
118
|
+
raise PathError("the path does not exist in the document")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
__all__ = ["PathError", "get_path", "parse_path", "replace_at"]
|