crawlutron-safe-url 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawlutron_safe_url-0.5.0/.gitignore +38 -0
- crawlutron_safe_url-0.5.0/PKG-INFO +41 -0
- crawlutron_safe_url-0.5.0/README.md +31 -0
- crawlutron_safe_url-0.5.0/pyproject.toml +18 -0
- crawlutron_safe_url-0.5.0/src/safe_url/__init__.py +57 -0
- crawlutron_safe_url-0.5.0/src/safe_url/address_class.py +175 -0
- crawlutron_safe_url-0.5.0/src/safe_url/host_canonical.py +27 -0
- crawlutron_safe_url-0.5.0/src/safe_url/host_policy.py +219 -0
- crawlutron_safe_url-0.5.0/src/safe_url/mapped_value_errors.py +5 -0
- crawlutron_safe_url-0.5.0/src/safe_url/normalized_url.py +390 -0
- crawlutron_safe_url-0.5.0/src/safe_url/process_state.py +76 -0
- crawlutron_safe_url-0.5.0/src/safe_url/py.typed +0 -0
- crawlutron_safe_url-0.5.0/src/safe_url/registrable_domain.py +72 -0
- crawlutron_safe_url-0.5.0/src/safe_url/safe_regex.py +73 -0
- crawlutron_safe_url-0.5.0/src/safe_url/scope_match.py +33 -0
- crawlutron_safe_url-0.5.0/src/safe_url/url_redaction.py +161 -0
- crawlutron_safe_url-0.5.0/tests/test_public_api.py +170 -0
- crawlutron_safe_url-0.5.0/tests/test_standalone_import.py +32 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.pytest_cache/
|
|
4
|
+
report/
|
|
5
|
+
# Build artifacts from `python -m build` / editable installs (#26).
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
# Both spellings: a linked worktree reaches the venv through a `.venv` symlink,
|
|
9
|
+
# which the trailing-slash form does not match -- and a committed symlink into
|
|
10
|
+
# one checkout's venv is worse than no venv at all.
|
|
11
|
+
.venv/
|
|
12
|
+
.venv
|
|
13
|
+
.dynamic-task-ledger.json
|
|
14
|
+
*.egg-info/
|
|
15
|
+
.cache/
|
|
16
|
+
.mypy_cache/
|
|
17
|
+
.ruff_cache/
|
|
18
|
+
.gate_fanout/
|
|
19
|
+
.coverage
|
|
20
|
+
htmlcov/
|
|
21
|
+
# Session-analysis residue written by the cortexkit tooling. It is ephemeral
|
|
22
|
+
# per-session output, not project content, and its own managed block already
|
|
23
|
+
# declares `magic-context/` untracked -- the historian XML under it was
|
|
24
|
+
# committed anyway, which is how a directory full of transcript digests ended
|
|
25
|
+
# up in the tree this service is extracted from.
|
|
26
|
+
.cortexkit/
|
|
27
|
+
logs/
|
|
28
|
+
cache/
|
|
29
|
+
.env
|
|
30
|
+
tmp/
|
|
31
|
+
*.log
|
|
32
|
+
.DS_Store
|
|
33
|
+
|
|
34
|
+
.coverage.*
|
|
35
|
+
.opencode
|
|
36
|
+
.hardstand/
|
|
37
|
+
|
|
38
|
+
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: crawlutron-safe-url
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: SSRF-safe URL admission: NormalizedUrl, HostPolicy, IDNA canonicalization, registrable domain, private-target guard, credential-safe redaction
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.14
|
|
7
|
+
Requires-Dist: publicsuffixlist==1.1.2.20261101
|
|
8
|
+
Requires-Dist: regex==2026.9.29
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# crawlutron-safe-url
|
|
12
|
+
|
|
13
|
+
A URL type that *is* SSRF-safe admission.
|
|
14
|
+
|
|
15
|
+
Any service that accepts a URL -- a link shortener, a webhook receiver, a URL
|
|
16
|
+
expander, a social unfurler, a browser driver, an LLM tool -- needs "can I safely
|
|
17
|
+
fetch this?" answered by a constructor rather than by a hand-rolled
|
|
18
|
+
`urlparse` + `ipaddress` check. `NormalizedUrl` does that: host policy, IDNA
|
|
19
|
+
canonicalization, registrable domain, private-target guard, and credential-safe
|
|
20
|
+
redaction, in one admission step.
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from safe_url import NormalizedUrl, HostPolicy
|
|
24
|
+
|
|
25
|
+
policy = HostPolicy.from_strings("example.com,*.trusted.example", "*.ads.example")
|
|
26
|
+
url = NormalizedUrl("HTTPS://Example.COM:443/a/b/?utm_source=x#frag", policy=policy)
|
|
27
|
+
url.raw # 'https://example.com/a/b/'
|
|
28
|
+
url.host # 'example.com'
|
|
29
|
+
url.site # 'example.com'
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
A URL that fails admission is never constructed, so there is no later "did I check
|
|
33
|
+
this?" -- the type is the check. The package imports nothing from `crawlutron`:
|
|
34
|
+
`crawlutron-core` depends on it, never the reverse.
|
|
35
|
+
|
|
36
|
+
Install with the public-suffix dataset for registrable-domain lookups:
|
|
37
|
+
`pip install "crawlutron-safe-url[psl]"`.
|
|
38
|
+
|
|
39
|
+
## License
|
|
40
|
+
|
|
41
|
+
MIT. See the repository's `LICENSE`.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# crawlutron-safe-url
|
|
2
|
+
|
|
3
|
+
A URL type that *is* SSRF-safe admission.
|
|
4
|
+
|
|
5
|
+
Any service that accepts a URL -- a link shortener, a webhook receiver, a URL
|
|
6
|
+
expander, a social unfurler, a browser driver, an LLM tool -- needs "can I safely
|
|
7
|
+
fetch this?" answered by a constructor rather than by a hand-rolled
|
|
8
|
+
`urlparse` + `ipaddress` check. `NormalizedUrl` does that: host policy, IDNA
|
|
9
|
+
canonicalization, registrable domain, private-target guard, and credential-safe
|
|
10
|
+
redaction, in one admission step.
|
|
11
|
+
|
|
12
|
+
```python
|
|
13
|
+
from safe_url import NormalizedUrl, HostPolicy
|
|
14
|
+
|
|
15
|
+
policy = HostPolicy.from_strings("example.com,*.trusted.example", "*.ads.example")
|
|
16
|
+
url = NormalizedUrl("HTTPS://Example.COM:443/a/b/?utm_source=x#frag", policy=policy)
|
|
17
|
+
url.raw # 'https://example.com/a/b/'
|
|
18
|
+
url.host # 'example.com'
|
|
19
|
+
url.site # 'example.com'
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
A URL that fails admission is never constructed, so there is no later "did I check
|
|
23
|
+
this?" -- the type is the check. The package imports nothing from `crawlutron`:
|
|
24
|
+
`crawlutron-core` depends on it, never the reverse.
|
|
25
|
+
|
|
26
|
+
Install with the public-suffix dataset for registrable-domain lookups:
|
|
27
|
+
`pip install "crawlutron-safe-url[psl]"`.
|
|
28
|
+
|
|
29
|
+
## License
|
|
30
|
+
|
|
31
|
+
MIT. See the repository's `LICENSE`.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "crawlutron-safe-url"
|
|
7
|
+
version = "0.5.0"
|
|
8
|
+
description = "SSRF-safe URL admission: NormalizedUrl, HostPolicy, IDNA canonicalization, registrable domain, private-target guard, credential-safe redaction"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.14"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
dependencies = [
|
|
13
|
+
"regex==2026.9.29",
|
|
14
|
+
"publicsuffixlist==1.1.2.20261101",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[tool.hatch.build.targets.wheel]
|
|
18
|
+
packages = ["src/safe_url"]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""SSRF-safe URL admission: NormalizedUrl, HostPolicy, credential-safe redaction."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from safe_url.address_class import AddressClass, classify, is_loopback_bind
|
|
6
|
+
from safe_url.host_canonical import idna_host, punycode_host
|
|
7
|
+
from safe_url.host_policy import (
|
|
8
|
+
HostPolicy,
|
|
9
|
+
allow_private_targets,
|
|
10
|
+
host_egress_reason,
|
|
11
|
+
host_policy_reason,
|
|
12
|
+
reset_private_target_memo,
|
|
13
|
+
)
|
|
14
|
+
from safe_url.normalized_url import NormalizedUrl, admit_or_none, admit_with_reason, refusal_literals
|
|
15
|
+
from safe_url.registrable_domain import is_public_suffix, registrable_domain
|
|
16
|
+
from safe_url.safe_regex import (
|
|
17
|
+
DEFAULT_MATCH_TIMEOUT_SECONDS,
|
|
18
|
+
MAX_PATTERN_LENGTH,
|
|
19
|
+
BoundedPattern,
|
|
20
|
+
RegexTimeout,
|
|
21
|
+
UnsafePattern,
|
|
22
|
+
compile_untrusted,
|
|
23
|
+
match_bounded,
|
|
24
|
+
)
|
|
25
|
+
from safe_url.scope_match import scope_allows
|
|
26
|
+
from safe_url.url_redaction import is_secret_name, redact_url, redact_urls_in_text, redacted_marker
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"DEFAULT_MATCH_TIMEOUT_SECONDS",
|
|
30
|
+
"MAX_PATTERN_LENGTH",
|
|
31
|
+
"AddressClass",
|
|
32
|
+
"BoundedPattern",
|
|
33
|
+
"HostPolicy",
|
|
34
|
+
"NormalizedUrl",
|
|
35
|
+
"RegexTimeout",
|
|
36
|
+
"UnsafePattern",
|
|
37
|
+
"admit_or_none",
|
|
38
|
+
"admit_with_reason",
|
|
39
|
+
"allow_private_targets",
|
|
40
|
+
"classify",
|
|
41
|
+
"compile_untrusted",
|
|
42
|
+
"host_egress_reason",
|
|
43
|
+
"host_policy_reason",
|
|
44
|
+
"idna_host",
|
|
45
|
+
"is_loopback_bind",
|
|
46
|
+
"is_public_suffix",
|
|
47
|
+
"is_secret_name",
|
|
48
|
+
"match_bounded",
|
|
49
|
+
"punycode_host",
|
|
50
|
+
"redact_url",
|
|
51
|
+
"redact_urls_in_text",
|
|
52
|
+
"redacted_marker",
|
|
53
|
+
"refusal_literals",
|
|
54
|
+
"registrable_domain",
|
|
55
|
+
"reset_private_target_memo",
|
|
56
|
+
"scope_allows",
|
|
57
|
+
]
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Address classification: parse once, judge once, answer many questions.
|
|
2
|
+
Every IP verdict needs the same two steps (parse the string, unwrap an IPv4 inside
|
|
3
|
+
IPv6, read the range predicates), so the ranges are classified here and the questions asked elsewhere.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import ipaddress
|
|
9
|
+
from enum import StrEnum
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"AddressClass",
|
|
13
|
+
"classify",
|
|
14
|
+
"is_loopback_bind",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
# Ranges that ``ipaddress`` does not model as private and that still must not
|
|
18
|
+
# be reached by a public egress, nor bound by a service expecting to be local.
|
|
19
|
+
_CGNAT_NETWORK = ipaddress.ip_network("100.64.0.0/10")
|
|
20
|
+
_IPV6_UNIQUE_LOCAL_NETWORK = ipaddress.ip_network("fc00::/7")
|
|
21
|
+
_NAT64_NETWORKS = [
|
|
22
|
+
ipaddress.ip_network("64:ff9b::/96"),
|
|
23
|
+
# RFC 8215 local-use prefix; same fall-through hazard as the /96.
|
|
24
|
+
ipaddress.ip_network("64:ff9b:1::/48"),
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class AddressClass(StrEnum):
|
|
29
|
+
"""Which range an address falls in, or that it is not an address at all.
|
|
30
|
+
NOT_AN_ADDRESS is a first-class answer, not a failure: folding "could not
|
|
31
|
+
parse" into PUBLIC would silently invert every consumer's decision.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
PUBLIC = "public"
|
|
35
|
+
PRIVATE = "private"
|
|
36
|
+
LOOPBACK = "loopback"
|
|
37
|
+
LINK_LOCAL = "link-local"
|
|
38
|
+
CGNAT = "cgnat"
|
|
39
|
+
UNIQUE_LOCAL = "unique-local"
|
|
40
|
+
NAT64 = "nat64"
|
|
41
|
+
MULTICAST = "multicast"
|
|
42
|
+
UNSPECIFIED = "unspecified"
|
|
43
|
+
RESERVED = "reserved"
|
|
44
|
+
NOT_AN_ADDRESS = "not-an-address"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _resolver_numeric_part(part: str) -> int | None:
|
|
48
|
+
"""Parse one numeric IP component the way glibc-style resolvers do."""
|
|
49
|
+
if not part:
|
|
50
|
+
return None
|
|
51
|
+
lowered = part.lower()
|
|
52
|
+
if lowered.startswith("0x"):
|
|
53
|
+
digits = lowered.removeprefix("0x")
|
|
54
|
+
if not digits or any(c not in "0123456789abcdef" for c in digits):
|
|
55
|
+
return None
|
|
56
|
+
return int(digits, 16)
|
|
57
|
+
if len(part) > 1 and part.startswith("0"):
|
|
58
|
+
if any(c not in "0123456789" for c in part):
|
|
59
|
+
return None
|
|
60
|
+
return int(part, 8)
|
|
61
|
+
if not part.isdigit():
|
|
62
|
+
return None
|
|
63
|
+
return int(part, 10)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _resolver_numeric_ipv4(host: str) -> int | None:
|
|
67
|
+
"""Interpret *host* as IPv4 the way resolvers parse numeric forms: ipaddress
|
|
68
|
+
rejects bare integers, hex and short dotted quads that resolvers accept, so the
|
|
69
|
+
packed value is classified itself. Returns the 32-bit value, or None when not a numeric form.
|
|
70
|
+
"""
|
|
71
|
+
if not host or any(c not in "0123456789abcdefxABCDEF." for c in host):
|
|
72
|
+
return None
|
|
73
|
+
if "." not in host:
|
|
74
|
+
value = _resolver_numeric_part(host)
|
|
75
|
+
if value is None:
|
|
76
|
+
return None
|
|
77
|
+
# Oversized single integers wrap mod 2**32 on resolvers (verified
|
|
78
|
+
# against getaddrinfo); classify the effective address, not None.
|
|
79
|
+
return value & 0xFFFFFFFF
|
|
80
|
+
parts = host.split(".")
|
|
81
|
+
if not 2 <= len(parts) <= 4:
|
|
82
|
+
return None
|
|
83
|
+
values = []
|
|
84
|
+
for part in parts:
|
|
85
|
+
parsed = _resolver_numeric_part(part)
|
|
86
|
+
if parsed is None:
|
|
87
|
+
return None
|
|
88
|
+
values.append(parsed)
|
|
89
|
+
width = len(values)
|
|
90
|
+
for leading in values[:-1]:
|
|
91
|
+
if leading > 0xFF:
|
|
92
|
+
return None
|
|
93
|
+
last_limit = 1 << (8 * (5 - width))
|
|
94
|
+
if values[-1] >= last_limit:
|
|
95
|
+
return None
|
|
96
|
+
result = 0
|
|
97
|
+
for index, value in enumerate(values[:-1]):
|
|
98
|
+
result |= value << (8 * (3 - index))
|
|
99
|
+
return result | values[-1]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _embedded_ipv4(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> ipaddress.IPv4Address | None:
|
|
103
|
+
"""Unwrap an IPv4 hidden inside an IPv6 wrapper: IPv4-mapped, 6to4 and Teredo
|
|
104
|
+
all reach an IPv4 target, so the verdict must be on the embedded IPv4, else a
|
|
105
|
+
wrapper hides what the range check rejects. None (native IPv6, bare IPv4) = classify ``addr`` as-is.
|
|
106
|
+
"""
|
|
107
|
+
if not isinstance(addr, ipaddress.IPv6Address):
|
|
108
|
+
return None
|
|
109
|
+
embedded = addr.ipv4_mapped or addr.sixtofour
|
|
110
|
+
if embedded is None and addr.teredo is not None:
|
|
111
|
+
# Teredo exposes (server, client); the client is the reachable target.
|
|
112
|
+
embedded = addr.teredo[1]
|
|
113
|
+
return embedded
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _parse(
|
|
117
|
+
host: str | ipaddress.IPv4Address | ipaddress.IPv6Address,
|
|
118
|
+
) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
|
119
|
+
"""Return the address *host* denotes, or None if it denotes none.
|
|
120
|
+
An already-parsed address returns unchanged, so a caller holding one does
|
|
121
|
+
not re-parse and cannot reach a different answer than a string holder.
|
|
122
|
+
"""
|
|
123
|
+
if isinstance(host, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
|
|
124
|
+
return host
|
|
125
|
+
try:
|
|
126
|
+
return ipaddress.ip_address(host)
|
|
127
|
+
except ValueError:
|
|
128
|
+
# Resolver-numeric forms ipaddress rejects: classify the packed value
|
|
129
|
+
# with the same predicates.
|
|
130
|
+
numeric = _resolver_numeric_ipv4(host)
|
|
131
|
+
return ipaddress.ip_address(numeric) if numeric is not None else None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def classify(host: str | ipaddress.IPv4Address | ipaddress.IPv6Address) -> AddressClass:
|
|
135
|
+
"""Return the range *host* falls in -- the one place an address is parsed,
|
|
136
|
+
unwrapped and read against a range. Precedence is the order of the checks below
|
|
137
|
+
(first match wins, PUBLIC is the fallback); private/reserved are both "not PUBLIC".
|
|
138
|
+
"""
|
|
139
|
+
candidate = host.strip() if isinstance(host, str) else host
|
|
140
|
+
address = _parse(candidate)
|
|
141
|
+
if address is None:
|
|
142
|
+
return AddressClass.NOT_AN_ADDRESS
|
|
143
|
+
classified = _embedded_ipv4(address) or address
|
|
144
|
+
if classified.is_loopback:
|
|
145
|
+
return AddressClass.LOOPBACK
|
|
146
|
+
if classified.is_unspecified:
|
|
147
|
+
return AddressClass.UNSPECIFIED
|
|
148
|
+
if classified.is_link_local:
|
|
149
|
+
return AddressClass.LINK_LOCAL
|
|
150
|
+
if classified.is_multicast:
|
|
151
|
+
return AddressClass.MULTICAST
|
|
152
|
+
if isinstance(classified, ipaddress.IPv4Address) and classified in _CGNAT_NETWORK:
|
|
153
|
+
return AddressClass.CGNAT
|
|
154
|
+
if isinstance(classified, ipaddress.IPv6Address) and classified in _IPV6_UNIQUE_LOCAL_NETWORK:
|
|
155
|
+
return AddressClass.UNIQUE_LOCAL
|
|
156
|
+
if isinstance(classified, ipaddress.IPv6Address) and any(
|
|
157
|
+
classified in network for network in _NAT64_NETWORKS
|
|
158
|
+
):
|
|
159
|
+
return AddressClass.NAT64
|
|
160
|
+
if classified.is_reserved:
|
|
161
|
+
return AddressClass.RESERVED
|
|
162
|
+
if classified.is_private:
|
|
163
|
+
return AddressClass.PRIVATE
|
|
164
|
+
return AddressClass.PUBLIC
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def is_loopback_bind(host: str) -> bool:
|
|
168
|
+
"""Return whether a listen address keeps the service on this machine.
|
|
169
|
+
Asked separately from the egress question: a private-range address is LAN-reachable,
|
|
170
|
+
so reusing egress would call it local. 127.0.0.0/8, ::1 and localhost qualify; unclassifiable = reachable.
|
|
171
|
+
"""
|
|
172
|
+
candidate = (host or "").strip()
|
|
173
|
+
if candidate.lower() == "localhost":
|
|
174
|
+
return True
|
|
175
|
+
return classify(candidate) is AddressClass.LOOPBACK
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""The one IDNA host grammar: every host that crosses this codebase passes it.
|
|
2
|
+
|
|
3
|
+
Admission refuses hosts it cannot encode; matching sites catch and passthrough.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
__all__ = ["idna_host", "punycode_host"]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def punycode_host(value: str) -> str:
|
|
12
|
+
"""The one IDNA encode: punycode, lower, strip the trailing FQDN dot.
|
|
13
|
+
Raises ``UnicodeError`` when the host cannot be encoded; callers own the
|
|
14
|
+
posture (refuse at admission, passthrough for policy matching)."""
|
|
15
|
+
return value.encode("idna").decode("ascii").lower().rstrip(".")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def idna_host(host: str) -> str:
|
|
19
|
+
"""Lower, strip the trailing FQDN dot, punycode, lower. Raises ``ValueError``
|
|
20
|
+
when the host cannot be encoded -- the caller owns its posture (refuse at
|
|
21
|
+
admission, passthrough for policy matching).
|
|
22
|
+
"""
|
|
23
|
+
value = host.strip().lower().rstrip(".")
|
|
24
|
+
try:
|
|
25
|
+
return punycode_host(value)
|
|
26
|
+
except UnicodeError as error:
|
|
27
|
+
raise ValueError("invalid_idn_host") from error
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Which hosts a crawl may reach, independent of any one URL."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from functools import lru_cache
|
|
9
|
+
from urllib.parse import urlparse
|
|
10
|
+
|
|
11
|
+
import safe_url.process_state as process_state
|
|
12
|
+
from safe_url.address_class import AddressClass, classify
|
|
13
|
+
from safe_url.host_canonical import punycode_host
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _normalize_policy_host(host: str) -> str:
|
|
17
|
+
value = host.strip().lower().rstrip(".")
|
|
18
|
+
if value.startswith("[") and value.endswith("]"):
|
|
19
|
+
value = value[1:-1]
|
|
20
|
+
try:
|
|
21
|
+
return punycode_host(value)
|
|
22
|
+
except UnicodeError:
|
|
23
|
+
return value
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _policy_host_entry(entry: str) -> str:
|
|
27
|
+
value = entry.strip()
|
|
28
|
+
if not value:
|
|
29
|
+
return ""
|
|
30
|
+
parsed = urlparse(value if "://" in value else f"//{value}")
|
|
31
|
+
host = parsed.hostname or value.split("/", 1)[0].split(":", 1)[0]
|
|
32
|
+
host = host.removeprefix("*.")
|
|
33
|
+
return _normalize_policy_host(host)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@lru_cache(maxsize=256)
|
|
37
|
+
def _policy_hosts(raw: str) -> tuple[str, ...]:
|
|
38
|
+
return tuple(entry for item in raw.split(",") if (entry := _policy_host_entry(item)))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class HostPolicy:
|
|
43
|
+
allowed_hosts: tuple[str, ...]
|
|
44
|
+
denied_hosts: tuple[str, ...]
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
def unrestricted(cls) -> HostPolicy:
|
|
48
|
+
return cls((), ())
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def from_strings(cls, allowed_hosts: str, denied_hosts: str) -> HostPolicy:
|
|
52
|
+
return cls(_policy_hosts(allowed_hosts), _policy_hosts(denied_hosts))
|
|
53
|
+
|
|
54
|
+
@classmethod
|
|
55
|
+
def from_settings(cls, settings: object) -> HostPolicy:
|
|
56
|
+
return cls.from_strings(
|
|
57
|
+
str(getattr(settings, "allowed_hosts", "") or ""),
|
|
58
|
+
str(getattr(settings, "denied_hosts", "") or ""),
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
def __post_init__(self) -> None:
|
|
62
|
+
object.__setattr__(
|
|
63
|
+
self,
|
|
64
|
+
"allowed_hosts",
|
|
65
|
+
tuple(_policy_host_entry(entry) for entry in self.allowed_hosts if _policy_host_entry(entry)),
|
|
66
|
+
)
|
|
67
|
+
object.__setattr__(
|
|
68
|
+
self,
|
|
69
|
+
"denied_hosts",
|
|
70
|
+
tuple(_policy_host_entry(entry) for entry in self.denied_hosts if _policy_host_entry(entry)),
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def host_policy_reason(host: str, *, policy: HostPolicy) -> str | None:
|
|
75
|
+
"""Why this host is refused, or None when the policy admits it. Hosts here are
|
|
76
|
+
admitted values (punycode via admission) or already-normalized config entries."""
|
|
77
|
+
if any(host == entry or host.endswith(f".{entry}") for entry in policy.denied_hosts):
|
|
78
|
+
return "host_denied"
|
|
79
|
+
if policy.allowed_hosts and not any(
|
|
80
|
+
host == entry or host.endswith(f".{entry}") for entry in policy.allowed_hosts
|
|
81
|
+
):
|
|
82
|
+
return "host_not_allowed"
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
#: (raw env value, parsed decision) for CRAWLUTRON_ALLOW_PRIVATE_TARGETS.
|
|
87
|
+
#: Dict keyed by the raw value; unchanged value hits the cache.
|
|
88
|
+
_PRIVATE_TARGET_CACHE: dict[str | None, bool] = {}
|
|
89
|
+
|
|
90
|
+
#: Hosts that name this machine rather than a site on the internet.
|
|
91
|
+
_PRIVATE_HOSTS = frozenset({"localhost", "host.docker.internal"})
|
|
92
|
+
|
|
93
|
+
#: Spelling variants that enable the private-target escape hatch.
|
|
94
|
+
_ON_SPELLINGS = frozenset({"1", "true", "yes", "on"})
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _private_target_default() -> bool:
|
|
98
|
+
return False
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _parse_private_target_env(raw: str | None) -> bool:
|
|
102
|
+
"""Parse the raw env value into a private-target decision.
|
|
103
|
+
|
|
104
|
+
Garbage warns and falls back to the default; unset/on/off parse cleanly.
|
|
105
|
+
"""
|
|
106
|
+
if raw is None or not raw.strip():
|
|
107
|
+
return False
|
|
108
|
+
lowered = raw.strip().lower()
|
|
109
|
+
if lowered in _ON_SPELLINGS:
|
|
110
|
+
return True
|
|
111
|
+
if lowered in {"0", "false", "no", "off"}:
|
|
112
|
+
return False
|
|
113
|
+
logging.getLogger("safe_url.host_policy").warning(
|
|
114
|
+
"invalid_bool_env for CRAWLUTRON_ALLOW_PRIVATE_TARGETS: %r (using default %r)",
|
|
115
|
+
raw,
|
|
116
|
+
False,
|
|
117
|
+
extra={
|
|
118
|
+
"env_name": "CRAWLUTRON_ALLOW_PRIVATE_TARGETS",
|
|
119
|
+
"env_value": raw,
|
|
120
|
+
"fallback": False,
|
|
121
|
+
},
|
|
122
|
+
)
|
|
123
|
+
return False
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _private_target_warning(raw: str | None) -> str:
|
|
127
|
+
return (
|
|
128
|
+
"SSRF private-target guard DISABLED by CRAWLUTRON_ALLOW_PRIVATE_TARGETS=%r: "
|
|
129
|
+
"caller-supplied URLs may now reach private, loopback, and link-local "
|
|
130
|
+
"addresses, including cloud metadata endpoints. Unset it, or set it to 0, "
|
|
131
|
+
"outside local development."
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _decide_private_targets() -> bool:
|
|
136
|
+
"""Read the private-target escape hatch from env (inverted default).
|
|
137
|
+
|
|
138
|
+
The raw env string is the cache key: an unchanged value skips the parse.
|
|
139
|
+
"""
|
|
140
|
+
raw = os.environ.get("CRAWLUTRON_ALLOW_PRIVATE_TARGETS")
|
|
141
|
+
if raw in _PRIVATE_TARGET_CACHE:
|
|
142
|
+
return _PRIVATE_TARGET_CACHE[raw]
|
|
143
|
+
decision = _parse_private_target_env(raw)
|
|
144
|
+
_PRIVATE_TARGET_CACHE[raw] = decision
|
|
145
|
+
if decision:
|
|
146
|
+
logging.getLogger("safe_url.host_policy").warning(
|
|
147
|
+
_private_target_warning(raw),
|
|
148
|
+
extra={"env_name": "CRAWLUTRON_ALLOW_PRIVATE_TARGETS", "env_value": raw},
|
|
149
|
+
)
|
|
150
|
+
return decision
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def allow_private_targets() -> bool:
|
|
154
|
+
"""Memoized read of the private-target escape hatch (inverted default).
|
|
155
|
+
|
|
156
|
+
Host verdicts are a pure function of the host and never go stale; the env
|
|
157
|
+
decision memo warns once per distinct value, not per call.
|
|
158
|
+
"""
|
|
159
|
+
return _decide_private_targets()
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def reset_private_target_memo() -> None:
|
|
163
|
+
"""Drop the memoized env decision (tests that count parse calls).
|
|
164
|
+
|
|
165
|
+
Host verdicts are a pure function of the host and never go stale, so the
|
|
166
|
+
verdict memo deliberately is not touched here.
|
|
167
|
+
"""
|
|
168
|
+
_PRIVATE_TARGET_CACHE.clear()
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _reset() -> None:
|
|
172
|
+
"""Drop the memoized env decision (registry route, conftest)."""
|
|
173
|
+
reset_private_target_memo()
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _has() -> bool:
|
|
177
|
+
"""Whether an env decision is memoized right now."""
|
|
178
|
+
return bool(_PRIVATE_TARGET_CACHE)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
process_state.register(__name__, _reset, _has)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@lru_cache(maxsize=8192)
|
|
185
|
+
def _host_is_public(host: str) -> bool:
|
|
186
|
+
"""SSRF verdict for one canonical host (bounded memo, pure function).
|
|
187
|
+
|
|
188
|
+
Deterministic per host and guarded-call-independent, so entries never go stale;
|
|
189
|
+
the bounded memo caps attacker-influenced host floods from growing memory.
|
|
190
|
+
"""
|
|
191
|
+
if host in _PRIVATE_HOSTS:
|
|
192
|
+
return False
|
|
193
|
+
if "%" in host:
|
|
194
|
+
# A zone id is never a routable target: refuse unconditionally rather
|
|
195
|
+
# than trust the stdlib's range answer, which has moved between versions.
|
|
196
|
+
return False
|
|
197
|
+
# A host that is not an address is a name, judged against ``is_public_ip_value``
|
|
198
|
+
# once the resolver answers; only a literal IP in a rejected range is refused.
|
|
199
|
+
return classify(host) in (AddressClass.NOT_AN_ADDRESS, AddressClass.PUBLIC)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def host_egress_reason(host: str) -> str | None:
|
|
203
|
+
"""Why this host may not be reached, or None when the SSRF guard admits it.
|
|
204
|
+
|
|
205
|
+
Separate from ``host_policy_reason``: that asks "on the list?", this "can we leave?".
|
|
206
|
+
The list check runs first, so an unlisted-but-private host answers ``host_not_allowed``.
|
|
207
|
+
"""
|
|
208
|
+
if allow_private_targets() or _host_is_public(host):
|
|
209
|
+
return None
|
|
210
|
+
return "private_network_target"
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
__all__ = [
|
|
214
|
+
"HostPolicy",
|
|
215
|
+
"allow_private_targets",
|
|
216
|
+
"host_egress_reason",
|
|
217
|
+
"host_policy_reason",
|
|
218
|
+
"reset_private_target_memo",
|
|
219
|
+
]
|