domainhack 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- domainhack/__init__.py +5 -0
- domainhack/__main__.py +5 -0
- domainhack/adapters/__init__.py +0 -0
- domainhack/adapters/_circuit.py +105 -0
- domainhack/adapters/_http.py +22 -0
- domainhack/adapters/_registration.py +128 -0
- domainhack/adapters/_stderr.py +16 -0
- domainhack/adapters/_text_sink.py +33 -0
- domainhack/adapters/_throttle.py +150 -0
- domainhack/adapters/brand_data.py +49 -0
- domainhack/adapters/brand_report_render.py +222 -0
- domainhack/adapters/cached_registrar.py +340 -0
- domainhack/adapters/composite_writer.py +27 -0
- domainhack/adapters/console_writer.py +77 -0
- domainhack/adapters/csv_writer.py +79 -0
- domainhack/adapters/dns_confirming_writer.py +72 -0
- domainhack/adapters/dns_resolver.py +120 -0
- domainhack/adapters/file_word_source.py +51 -0
- domainhack/adapters/iana_tlds.py +110 -0
- domainhack/adapters/json_writer.py +51 -0
- domainhack/adapters/pacing.py +46 -0
- domainhack/adapters/rdap_bootstrap.py +223 -0
- domainhack/adapters/rdap_registrar.py +328 -0
- domainhack/adapters/registrar_catalog.py +120 -0
- domainhack/adapters/registrar_router.py +72 -0
- domainhack/adapters/registry_sources.py +192 -0
- domainhack/adapters/tqdm_progress.py +101 -0
- domainhack/adapters/whois_registrar.py +386 -0
- domainhack/cli/__init__.py +0 -0
- domainhack/cli/app.py +800 -0
- domainhack/cli/name_cmd.py +222 -0
- domainhack/cli/tlds_cmd.py +96 -0
- domainhack/data/hack_words.json +20 -0
- domainhack/data/iana_tlds.txt +1438 -0
- domainhack/data/rdap_dns.json +1 -0
- domainhack/data/registry_sources.json +2367 -0
- domainhack/data/second_level.json +28 -0
- domainhack/data/tld_presets.json +7 -0
- domainhack/domain/__init__.py +0 -0
- domainhack/domain/entities.py +287 -0
- domainhack/domain/label_rules.py +96 -0
- domainhack/domain/parking.py +81 -0
- domainhack/ports/__init__.py +0 -0
- domainhack/ports/dns_lookup.py +11 -0
- domainhack/ports/known_tlds.py +21 -0
- domainhack/ports/progress.py +32 -0
- domainhack/ports/registrar.py +27 -0
- domainhack/ports/result_cache.py +19 -0
- domainhack/ports/result_writer.py +13 -0
- domainhack/ports/word_source.py +9 -0
- domainhack/py.typed +0 -0
- domainhack/usecases/__init__.py +0 -0
- domainhack/usecases/brand_candidates.py +195 -0
- domainhack/usecases/brand_report.py +271 -0
- domainhack/usecases/check_domains.py +287 -0
- domainhack/usecases/confirm_dns.py +64 -0
- domainhack/usecases/estimate_run.py +80 -0
- domainhack/usecases/filter_words.py +37 -0
- domainhack/usecases/generate_range.py +107 -0
- domainhack/usecases/lanes.py +193 -0
- domainhack/usecases/rank_candidates.py +106 -0
- domainhack-0.2.0.dist-info/METADATA +501 -0
- domainhack-0.2.0.dist-info/RECORD +67 -0
- domainhack-0.2.0.dist-info/WHEEL +5 -0
- domainhack-0.2.0.dist-info/entry_points.txt +2 -0
- domainhack-0.2.0.dist-info/licenses/LICENSE +21 -0
- domainhack-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Resolve a TLD to its RDAP base URL.
|
|
2
|
+
|
|
3
|
+
Order: denylist (servers whose answers this tool cannot use) -> overrides (working
|
|
4
|
+
servers missing from IANA's bootstrap) -> IANA bootstrap file
|
|
5
|
+
(https://data.iana.org/rdap/dns.json), fetched lazily and cached on disk
|
|
6
|
+
for ``ttl_seconds`` -> the snapshot of that file bundled with the package
|
|
7
|
+
(``domainhack/data/rdap_dns.json``).
|
|
8
|
+
|
|
9
|
+
The bundled snapshot makes the backend choice independent of the network:
|
|
10
|
+
offline, every TLD resolves exactly as it did when the snapshot was taken,
|
|
11
|
+
and a fresh bootstrap only adds or updates entries on top of it. When the
|
|
12
|
+
refresh fails, one warning goes to stderr and the snapshot alone is used; a
|
|
13
|
+
stale on-disk cache is deliberately *not* used, so offline runs are
|
|
14
|
+
deterministic whatever happens to be in ``~/.cache``.
|
|
15
|
+
|
|
16
|
+
Refresh the snapshot by downloading the IANA file and re-serialising it
|
|
17
|
+
minified (``json.dump(data, f, separators=(",", ":"))``) into
|
|
18
|
+
``src/domainhack/data/rdap_dns.json``. It is kept whole (about 32 KB) so any
|
|
19
|
+
TLD, not just the catalog's, resolves offline exactly as online.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import sys
|
|
27
|
+
import time
|
|
28
|
+
from collections.abc import Callable, Mapping
|
|
29
|
+
from importlib import resources
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
import httpx
|
|
34
|
+
|
|
35
|
+
from domainhack.adapters._http import identity_headers
|
|
36
|
+
from domainhack.adapters.registry_sources import load_registry_sources
|
|
37
|
+
|
|
38
|
+
IANA_BOOTSTRAP_URL = "https://data.iana.org/rdap/dns.json"
|
|
39
|
+
DEFAULT_TTL_SECONDS: float = 24 * 60 * 60
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# Registries missing from the IANA bootstrap (verified live) and RDAP servers
|
|
43
|
+
# whose answers this tool cannot use live in data/registry_sources.json. "to" is
|
|
44
|
+
# pinned there too so the most important TLD works even if the bootstrap fetch
|
|
45
|
+
# fails; .gg and .la are checked over WHOIS instead (their RDAP replies did not
|
|
46
|
+
# distinguish registered from unregistered names when verified).
|
|
47
|
+
RDAP_OVERRIDES: Mapping[str, str] = load_registry_sources().rdap_overrides()
|
|
48
|
+
RDAP_DENYLIST: frozenset[str] = load_registry_sources().rdap_denylist
|
|
49
|
+
|
|
50
|
+
# Aggregators/redirectors are never a source of truth (rdap.org says
|
|
51
|
+
# "No RDAP service" for .io, which would read as AVAILABLE).
|
|
52
|
+
_DENIED_HOSTS: frozenset[str] = frozenset({"rdap.org", "www.rdap.org"})
|
|
53
|
+
|
|
54
|
+
Fetcher = Callable[[], bytes]
|
|
55
|
+
SnapshotLoader = Callable[[], Any]
|
|
56
|
+
Warn = Callable[[str], None]
|
|
57
|
+
|
|
58
|
+
BUNDLED_SNAPSHOT = "data/rdap_dns.json"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def load_bundled_snapshot() -> Any:
|
|
62
|
+
"""The IANA bootstrap document shipped as package data."""
|
|
63
|
+
resource = resources.files("domainhack").joinpath(BUNDLED_SNAPSHOT)
|
|
64
|
+
return json.loads(resource.read_text(encoding="utf-8"))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def snapshot_date(data: Any) -> str:
|
|
68
|
+
"""``YYYY-MM-DD`` from a bootstrap document's ``publication`` field, or ``unknown``."""
|
|
69
|
+
publication = data.get("publication") if isinstance(data, dict) else None
|
|
70
|
+
return publication[:10] if isinstance(publication, str) and publication else "unknown"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _stderr_warn(message: str) -> None:
|
|
74
|
+
print(f"warning: {message}", file=sys.stderr)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def default_bootstrap_cache_path() -> Path:
|
|
78
|
+
"""Return ``$XDG_CACHE_HOME/domainhack/rdap_dns.json`` (or ``~/.cache/...``)."""
|
|
79
|
+
base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
|
|
80
|
+
return Path(base) / "domainhack" / "rdap_dns.json"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def fetch_iana_bootstrap(timeout: float = 10.0) -> bytes:
|
|
84
|
+
response = httpx.get(
|
|
85
|
+
IANA_BOOTSTRAP_URL, headers=identity_headers(), timeout=timeout, follow_redirects=True
|
|
86
|
+
)
|
|
87
|
+
response.raise_for_status()
|
|
88
|
+
return response.content
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def parse_bootstrap(data: Any) -> dict[str, str]:
|
|
92
|
+
"""Map each TLD in an RFC 9224 bootstrap document to its preferred base URL."""
|
|
93
|
+
result: dict[str, str] = {}
|
|
94
|
+
services = data.get("services", []) if isinstance(data, dict) else []
|
|
95
|
+
for service in services:
|
|
96
|
+
if not (isinstance(service, list) and len(service) >= 2):
|
|
97
|
+
continue
|
|
98
|
+
tlds, urls = service[0], service[1]
|
|
99
|
+
if not (isinstance(tlds, list) and isinstance(urls, list)):
|
|
100
|
+
continue
|
|
101
|
+
url = _pick_url([u for u in urls if isinstance(u, str)])
|
|
102
|
+
if url is None:
|
|
103
|
+
continue
|
|
104
|
+
for tld in tlds:
|
|
105
|
+
if isinstance(tld, str):
|
|
106
|
+
result[tld.lower()] = url
|
|
107
|
+
return result
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _pick_url(urls: list[str]) -> str | None:
|
|
111
|
+
usable = [u for u in urls if httpx.URL(u).host not in _DENIED_HOSTS]
|
|
112
|
+
https = [u for u in usable if u.startswith("https://")]
|
|
113
|
+
candidates = https or usable
|
|
114
|
+
if not candidates:
|
|
115
|
+
return None
|
|
116
|
+
chosen = candidates[0]
|
|
117
|
+
return chosen if chosen.endswith("/") else chosen + "/"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class RdapBootstrap:
|
|
121
|
+
"""TLD -> RDAP base URL resolver with a lazily loaded, disk-cached IANA file.
|
|
122
|
+
|
|
123
|
+
Lookups go overrides -> fresh bootstrap (disk cache younger than the TTL,
|
|
124
|
+
or a new fetch) -> bundled snapshot. A failed fetch emits one warning
|
|
125
|
+
through ``warn`` and leaves the snapshot as the only source. The fetch is
|
|
126
|
+
attempted at most once per instance, so a network outage does not cost
|
|
127
|
+
one timeout per TLD.
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
def __init__(
|
|
131
|
+
self,
|
|
132
|
+
cache_path: Path | None = None,
|
|
133
|
+
ttl_seconds: float = DEFAULT_TTL_SECONDS,
|
|
134
|
+
fetcher: Fetcher | None = None,
|
|
135
|
+
clock: Callable[[], float] = time.time,
|
|
136
|
+
overrides: Mapping[str, str] = RDAP_OVERRIDES,
|
|
137
|
+
denylist: frozenset[str] = RDAP_DENYLIST,
|
|
138
|
+
snapshot: SnapshotLoader = load_bundled_snapshot,
|
|
139
|
+
warn: Warn = _stderr_warn,
|
|
140
|
+
) -> None:
|
|
141
|
+
self._cache_path = cache_path if cache_path is not None else default_bootstrap_cache_path()
|
|
142
|
+
self._ttl = ttl_seconds
|
|
143
|
+
self._fetcher: Fetcher = fetcher if fetcher is not None else fetch_iana_bootstrap
|
|
144
|
+
self._clock = clock
|
|
145
|
+
self._overrides = {k.lower(): v for k, v in overrides.items()}
|
|
146
|
+
self._denylist = frozenset(t.lower() for t in denylist)
|
|
147
|
+
self._snapshot_loader = snapshot
|
|
148
|
+
self._warn = warn
|
|
149
|
+
self._services: dict[str, str] | None = None
|
|
150
|
+
|
|
151
|
+
def base_url_for(self, tld: str) -> str | None:
|
|
152
|
+
"""The RDAP base URL for ``tld``, or None.
|
|
153
|
+
|
|
154
|
+
A multi-label suffix (``com.ar``) is served by the registry of its
|
|
155
|
+
top-level domain (``ar`` -> https://rdap.nic.ar/) unless it has an
|
|
156
|
+
entry of its own; the query still names the whole domain
|
|
157
|
+
(``{base}domain/sumanda.com.ar``).
|
|
158
|
+
"""
|
|
159
|
+
key = tld.lower().lstrip(".")
|
|
160
|
+
url = self._resolve(key)
|
|
161
|
+
if url is None and "." in key:
|
|
162
|
+
url = self._resolve(key.rsplit(".", 1)[1])
|
|
163
|
+
return url
|
|
164
|
+
|
|
165
|
+
def _resolve(self, key: str) -> str | None:
|
|
166
|
+
if key in self._denylist:
|
|
167
|
+
return None
|
|
168
|
+
if key in self._overrides:
|
|
169
|
+
return self._overrides[key]
|
|
170
|
+
return self._load().get(key)
|
|
171
|
+
|
|
172
|
+
def known_tlds(self) -> set[str]:
|
|
173
|
+
tlds = set(self._overrides) | set(self._load())
|
|
174
|
+
return tlds - self._denylist
|
|
175
|
+
|
|
176
|
+
def _load(self) -> dict[str, str]:
|
|
177
|
+
if self._services is None:
|
|
178
|
+
snapshot_data = self._snapshot_loader()
|
|
179
|
+
services = parse_bootstrap(snapshot_data)
|
|
180
|
+
services.update(self._load_fresh(snapshot_date(snapshot_data)))
|
|
181
|
+
self._services = services
|
|
182
|
+
return self._services
|
|
183
|
+
|
|
184
|
+
def _load_fresh(self, snapshot_day: str) -> dict[str, str]:
|
|
185
|
+
"""Entries from a fresh cache or a new fetch; {} (with a warning) on failure."""
|
|
186
|
+
cached = self._read_cache()
|
|
187
|
+
if cached is not None and self._clock() - cached[0] <= self._ttl:
|
|
188
|
+
return parse_bootstrap(cached[1])
|
|
189
|
+
|
|
190
|
+
try:
|
|
191
|
+
raw = self._fetcher()
|
|
192
|
+
data = json.loads(raw)
|
|
193
|
+
services = parse_bootstrap(data)
|
|
194
|
+
if not services:
|
|
195
|
+
raise ValueError("empty RDAP bootstrap")
|
|
196
|
+
except (httpx.HTTPError, OSError, ValueError) as exc:
|
|
197
|
+
reason = str(exc) or type(exc).__name__
|
|
198
|
+
self._warn(
|
|
199
|
+
f"could not refresh RDAP bootstrap ({reason}); "
|
|
200
|
+
f"using bundled snapshot from {snapshot_day}"
|
|
201
|
+
)
|
|
202
|
+
return {}
|
|
203
|
+
|
|
204
|
+
self._write_cache(data)
|
|
205
|
+
return services
|
|
206
|
+
|
|
207
|
+
def _read_cache(self) -> tuple[float, Any] | None:
|
|
208
|
+
try:
|
|
209
|
+
wrapper = json.loads(self._cache_path.read_text(encoding="utf-8"))
|
|
210
|
+
return float(wrapper["fetched_at"]), wrapper["data"]
|
|
211
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
212
|
+
return None
|
|
213
|
+
|
|
214
|
+
def _write_cache(self, data: Any) -> None:
|
|
215
|
+
try:
|
|
216
|
+
self._cache_path.parent.mkdir(parents=True, exist_ok=True)
|
|
217
|
+
tmp = self._cache_path.with_suffix(".tmp")
|
|
218
|
+
tmp.write_text(
|
|
219
|
+
json.dumps({"fetched_at": self._clock(), "data": data}), encoding="utf-8"
|
|
220
|
+
)
|
|
221
|
+
tmp.replace(self._cache_path)
|
|
222
|
+
except OSError:
|
|
223
|
+
pass
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""RDAP (RFC 9082/9083) availability checks: GET {base}domain/{fqdn}."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import random as _random
|
|
7
|
+
import time
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from email.utils import parsedate_to_datetime
|
|
11
|
+
from typing import Any
|
|
12
|
+
from urllib.parse import quote
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
from domainhack.adapters._circuit import HostCircuitBreaker
|
|
17
|
+
from domainhack.adapters._http import identity_headers
|
|
18
|
+
from domainhack.adapters._registration import (
|
|
19
|
+
normalize_name,
|
|
20
|
+
normalize_nameservers,
|
|
21
|
+
normalize_statuses,
|
|
22
|
+
parse_datetime_utc,
|
|
23
|
+
)
|
|
24
|
+
from domainhack.adapters._throttle import (
|
|
25
|
+
DEFAULT_BACKOFF_BASE,
|
|
26
|
+
DEFAULT_BACKOFF_CAP,
|
|
27
|
+
DEFAULT_MAX_INTERVAL,
|
|
28
|
+
DEFAULT_THROTTLE,
|
|
29
|
+
HostThrottle,
|
|
30
|
+
full_jitter_backoff,
|
|
31
|
+
)
|
|
32
|
+
from domainhack.domain.entities import Availability, DomainCheckResult, DomainHack
|
|
33
|
+
from domainhack.domain.parking import parking_hint_for
|
|
34
|
+
from domainhack.ports.registrar import RegistrarClient
|
|
35
|
+
|
|
36
|
+
_ACCEPT = {"Accept": "application/rdap+json, application/json"}
|
|
37
|
+
_STATUS_TAKEN = 200
|
|
38
|
+
_STATUS_AVAILABLE = 404
|
|
39
|
+
_STATUS_TOO_MANY = 429
|
|
40
|
+
_STATUS_SERVER_ERROR = 500
|
|
41
|
+
_STATUS_UNAVAILABLE = 503
|
|
42
|
+
# Answers that tell us to send less: they slow the host's throttle down.
|
|
43
|
+
_SLOW_DOWN_STATUSES = frozenset({_STATUS_TOO_MANY, _STATUS_UNAVAILABLE})
|
|
44
|
+
# Transport errors worth one more try: timeouts and dropped connections.
|
|
45
|
+
_RETRYABLE_ERRORS = (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError)
|
|
46
|
+
DEFAULT_CONNECT_TIMEOUT = 5.0
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def parse_retry_after(value: str | None, now: float | None = None) -> float | None:
|
|
50
|
+
"""Parse a Retry-After header (delta-seconds or HTTP-date) into seconds."""
|
|
51
|
+
if not value:
|
|
52
|
+
return None
|
|
53
|
+
value = value.strip()
|
|
54
|
+
if value.isdigit():
|
|
55
|
+
return float(value)
|
|
56
|
+
try:
|
|
57
|
+
when = parsedate_to_datetime(value)
|
|
58
|
+
except (TypeError, ValueError):
|
|
59
|
+
return None
|
|
60
|
+
current = time.time() if now is None else now
|
|
61
|
+
return max(when.timestamp() - current, 0.0)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class RdapRegistrarClient(RegistrarClient):
|
|
65
|
+
"""Checks availability against one RDAP base URL.
|
|
66
|
+
|
|
67
|
+
The HTTP status decides: 404 -> AVAILABLE, 200 with a JSON domain object
|
|
68
|
+
whose ``ldhName`` matches -> TAKEN, anything else (429, 5xx, timeouts,
|
|
69
|
+
HTML 200 pages) -> ERROR. ``raw_title`` holds ``"HTTP <status>"``.
|
|
70
|
+
TAKEN results carry the domain's ``status`` values, its ``registration``
|
|
71
|
+
and ``expiration`` events, its nameservers (plus the parking service they
|
|
72
|
+
point to, if recognised) and the sponsoring registrar's name, when the
|
|
73
|
+
body has them; malformed parts are ignored.
|
|
74
|
+
|
|
75
|
+
429, 5xx, timeouts and dropped connections are retried up to
|
|
76
|
+
``max_retries`` times. A ``Retry-After`` header is honored (it defers the
|
|
77
|
+
host on the shared ``throttle``); a 429 whose ``Retry-After`` exceeds
|
|
78
|
+
``max_retry_after`` is not retried. Without the header the retry waits a
|
|
79
|
+
full-jitter exponential backoff, ``random(0, min(backoff_cap,
|
|
80
|
+
backoff_base * 2**n))``. A 429, a 503 or a timeout also slows the host's
|
|
81
|
+
throttle down; a real answer helps it recover.
|
|
82
|
+
|
|
83
|
+
A check that still fails after its retries counts as one failure for the
|
|
84
|
+
per-host circuit ``breaker`` (transport errors, 429, 5xx); while a host's
|
|
85
|
+
circuit is open its domains get an ERROR without any network call.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def __init__(
|
|
89
|
+
self,
|
|
90
|
+
base_url: str,
|
|
91
|
+
delay: float = 1.0,
|
|
92
|
+
timeout: float = 10.0,
|
|
93
|
+
client: httpx.Client | None = None,
|
|
94
|
+
*,
|
|
95
|
+
max_retries: int = 2,
|
|
96
|
+
max_retry_after: float = 30.0,
|
|
97
|
+
throttle: HostThrottle | None = None,
|
|
98
|
+
breaker: HostCircuitBreaker | None = None,
|
|
99
|
+
random: Callable[[], float] = _random.random,
|
|
100
|
+
backoff_base: float = DEFAULT_BACKOFF_BASE,
|
|
101
|
+
backoff_cap: float = DEFAULT_BACKOFF_CAP,
|
|
102
|
+
contact: str | None = None,
|
|
103
|
+
) -> None:
|
|
104
|
+
self._headers = {**_ACCEPT, **identity_headers(contact)}
|
|
105
|
+
self._base_url = base_url if base_url.endswith("/") else base_url + "/"
|
|
106
|
+
self._host = httpx.URL(self._base_url).host
|
|
107
|
+
self._delay = delay
|
|
108
|
+
self._timeout = timeout
|
|
109
|
+
self._owns_client = client is None
|
|
110
|
+
self._client = client or httpx.Client(
|
|
111
|
+
headers=self._headers,
|
|
112
|
+
timeout=httpx.Timeout(timeout, connect=min(DEFAULT_CONNECT_TIMEOUT, timeout)),
|
|
113
|
+
follow_redirects=True,
|
|
114
|
+
)
|
|
115
|
+
self._max_retries = max_retries
|
|
116
|
+
self._max_retry_after = max_retry_after
|
|
117
|
+
self._throttle = throttle if throttle is not None else DEFAULT_THROTTLE
|
|
118
|
+
self._breaker = breaker if breaker is not None else HostCircuitBreaker()
|
|
119
|
+
self._random = random
|
|
120
|
+
self._backoff_base = backoff_base
|
|
121
|
+
self._backoff_cap = backoff_cap
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def base_url(self) -> str:
|
|
125
|
+
return self._base_url
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def http_client(self) -> httpx.Client:
|
|
129
|
+
"""The underlying HTTP client: the injected one, or the one this instance owns."""
|
|
130
|
+
return self._client
|
|
131
|
+
|
|
132
|
+
def check_availability(self, domain: DomainHack) -> DomainCheckResult:
|
|
133
|
+
if not self._breaker.allow(self._host):
|
|
134
|
+
return _error(domain, self._breaker.skip_message(self._host))
|
|
135
|
+
# fqdn is already a validated ASCII name; quoting the path segment is
|
|
136
|
+
# defence in depth so no name can add path segments or a query string.
|
|
137
|
+
url = f"{self._base_url}domain/{quote(domain.fqdn, safe='')}"
|
|
138
|
+
# One check is one outcome: however many attempts it takes, it slows
|
|
139
|
+
# the host down at most once and counts at most once for the breaker.
|
|
140
|
+
slowed = False
|
|
141
|
+
attempt = 0
|
|
142
|
+
while True:
|
|
143
|
+
self._throttle.wait(self._host, self._delay)
|
|
144
|
+
try:
|
|
145
|
+
response = self._client.get(url, headers=self._headers)
|
|
146
|
+
except httpx.HTTPError as e:
|
|
147
|
+
if isinstance(e, httpx.TimeoutException) and not slowed:
|
|
148
|
+
slowed = True
|
|
149
|
+
self._throttle.slow_down(self._host)
|
|
150
|
+
if isinstance(e, _RETRYABLE_ERRORS) and attempt < self._max_retries:
|
|
151
|
+
self._backoff(attempt)
|
|
152
|
+
attempt += 1
|
|
153
|
+
continue
|
|
154
|
+
self._breaker.record_failure(self._host)
|
|
155
|
+
return _error(domain, f"RDAP request failed: {e or type(e).__name__}")
|
|
156
|
+
|
|
157
|
+
status = response.status_code
|
|
158
|
+
if status != _STATUS_TOO_MANY and status < _STATUS_SERVER_ERROR:
|
|
159
|
+
self._throttle.record_success(self._host)
|
|
160
|
+
self._breaker.record_success(self._host)
|
|
161
|
+
return self._interpret(domain, response)
|
|
162
|
+
|
|
163
|
+
if status in _SLOW_DOWN_STATUSES and not slowed:
|
|
164
|
+
slowed = True
|
|
165
|
+
self._throttle.slow_down(self._host)
|
|
166
|
+
retry_after = parse_retry_after(response.headers.get("Retry-After"))
|
|
167
|
+
if retry_after is not None:
|
|
168
|
+
# RFC 7480 §5.5: honor Retry-After, for every client of this host.
|
|
169
|
+
self._throttle.defer(self._host, min(retry_after, DEFAULT_MAX_INTERVAL))
|
|
170
|
+
may_retry = retry_after is None or retry_after <= self._max_retry_after
|
|
171
|
+
if may_retry and attempt < self._max_retries:
|
|
172
|
+
if retry_after is None:
|
|
173
|
+
self._backoff(attempt)
|
|
174
|
+
attempt += 1
|
|
175
|
+
continue
|
|
176
|
+
self._breaker.record_failure(self._host)
|
|
177
|
+
if status == _STATUS_TOO_MANY:
|
|
178
|
+
return _error(domain, "RDAP rate limited (429)", status)
|
|
179
|
+
return self._interpret(domain, response)
|
|
180
|
+
|
|
181
|
+
def _backoff(self, attempt: int) -> None:
|
|
182
|
+
"""Full-jitter exponential backoff before retry ``attempt`` (0-based).
|
|
183
|
+
|
|
184
|
+
The wait goes through the throttle, so it overlaps with (rather than
|
|
185
|
+
adds to) any slow-down already pending for the host.
|
|
186
|
+
"""
|
|
187
|
+
delay = full_jitter_backoff(
|
|
188
|
+
attempt, self._random, base=self._backoff_base, cap=self._backoff_cap
|
|
189
|
+
)
|
|
190
|
+
self._throttle.defer(self._host, delay)
|
|
191
|
+
|
|
192
|
+
def _interpret(self, domain: DomainHack, response: httpx.Response) -> DomainCheckResult:
|
|
193
|
+
status = response.status_code
|
|
194
|
+
title = f"HTTP {status}"
|
|
195
|
+
if status == _STATUS_AVAILABLE:
|
|
196
|
+
return DomainCheckResult(
|
|
197
|
+
domain=domain, availability=Availability.AVAILABLE, raw_title=title
|
|
198
|
+
)
|
|
199
|
+
if status != _STATUS_TAKEN:
|
|
200
|
+
return _error(domain, f"Unexpected RDAP status {status}", status)
|
|
201
|
+
|
|
202
|
+
try:
|
|
203
|
+
body: Any = response.json()
|
|
204
|
+
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
205
|
+
return _error(domain, "RDAP 200 response is not JSON", status)
|
|
206
|
+
if not isinstance(body, dict):
|
|
207
|
+
return _error(domain, "RDAP 200 response is not a JSON object", status)
|
|
208
|
+
|
|
209
|
+
ldh = body.get("ldhName")
|
|
210
|
+
if isinstance(ldh, str):
|
|
211
|
+
if ldh.rstrip(".").lower() == domain.fqdn.lower():
|
|
212
|
+
return _taken(domain, title, body)
|
|
213
|
+
return _error(domain, f"RDAP ldhName mismatch: {ldh!r}", status)
|
|
214
|
+
if body.get("objectClassName") == "domain":
|
|
215
|
+
return _taken(domain, title, body)
|
|
216
|
+
return _error(domain, "RDAP 200 response is not a domain object", status)
|
|
217
|
+
|
|
218
|
+
def close(self) -> None:
|
|
219
|
+
if self._owns_client:
|
|
220
|
+
self._client.close()
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _taken(domain: DomainHack, title: str, body: dict[str, Any]) -> DomainCheckResult:
|
|
224
|
+
nameservers = parse_rdap_nameservers(body)
|
|
225
|
+
return DomainCheckResult(
|
|
226
|
+
domain=domain,
|
|
227
|
+
availability=Availability.TAKEN,
|
|
228
|
+
raw_title=title,
|
|
229
|
+
statuses=parse_rdap_statuses(body),
|
|
230
|
+
expires_at=parse_rdap_expiration(body),
|
|
231
|
+
registered_at=parse_rdap_registration(body),
|
|
232
|
+
registrar=parse_rdap_registrar(body),
|
|
233
|
+
nameservers=nameservers,
|
|
234
|
+
parked_hint=parking_hint_for(nameservers),
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def parse_rdap_statuses(body: dict[str, Any]) -> tuple[str, ...]:
|
|
239
|
+
"""The domain's ``status`` array (RFC 9083 §4.6), normalized; ``()`` if absent or malformed."""
|
|
240
|
+
values = body.get("status")
|
|
241
|
+
if not isinstance(values, list):
|
|
242
|
+
return ()
|
|
243
|
+
return normalize_statuses(values)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _event_date(body: dict[str, Any], action: str) -> datetime | None:
|
|
247
|
+
"""The date of the first valid event with ``eventAction == action`` (RFC 9083 §4.5)."""
|
|
248
|
+
events = body.get("events")
|
|
249
|
+
if not isinstance(events, list):
|
|
250
|
+
return None
|
|
251
|
+
for event in events:
|
|
252
|
+
if not isinstance(event, dict):
|
|
253
|
+
continue
|
|
254
|
+
event_action = event.get("eventAction")
|
|
255
|
+
if not isinstance(event_action, str) or event_action.strip().lower() != action:
|
|
256
|
+
continue
|
|
257
|
+
when = parse_datetime_utc(event.get("eventDate"))
|
|
258
|
+
if when is not None:
|
|
259
|
+
return when
|
|
260
|
+
return None
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def parse_rdap_expiration(body: dict[str, Any]) -> datetime | None:
|
|
264
|
+
"""The date of the first valid ``expiration`` event, in UTC, or None."""
|
|
265
|
+
return _event_date(body, "expiration")
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def parse_rdap_registration(body: dict[str, Any]) -> datetime | None:
|
|
269
|
+
"""The date of the first valid ``registration`` event, in UTC, or None."""
|
|
270
|
+
return _event_date(body, "registration")
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def parse_rdap_nameservers(body: dict[str, Any]) -> tuple[str, ...]:
|
|
274
|
+
"""``ldhName`` of each ``nameservers`` entry (RFC 9083 §5.2), or ``()``.
|
|
275
|
+
|
|
276
|
+
Lowercase, without the trailing dot, deduplicated, in registry order;
|
|
277
|
+
malformed entries are skipped.
|
|
278
|
+
"""
|
|
279
|
+
entries = body.get("nameservers")
|
|
280
|
+
if not isinstance(entries, list):
|
|
281
|
+
return ()
|
|
282
|
+
return normalize_nameservers(
|
|
283
|
+
entry.get("ldhName") for entry in entries if isinstance(entry, dict)
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def parse_rdap_registrar(body: dict[str, Any]) -> str:
|
|
288
|
+
"""The name of the top-level entity with role ``registrar`` (RFC 9083 §5.1).
|
|
289
|
+
|
|
290
|
+
Its vCard ``fn`` (RFC 7095 jCard) when non-empty, else its ``handle``
|
|
291
|
+
(for gTLDs, the IANA registrar ID); ``""`` if there is none.
|
|
292
|
+
"""
|
|
293
|
+
entities = body.get("entities")
|
|
294
|
+
if not isinstance(entities, list):
|
|
295
|
+
return ""
|
|
296
|
+
for entity in entities:
|
|
297
|
+
if not isinstance(entity, dict):
|
|
298
|
+
continue
|
|
299
|
+
roles = entity.get("roles")
|
|
300
|
+
if not isinstance(roles, list) or not any(
|
|
301
|
+
isinstance(role, str) and role.strip().lower() == "registrar" for role in roles
|
|
302
|
+
):
|
|
303
|
+
continue
|
|
304
|
+
name = _vcard_fn(entity.get("vcardArray")) or normalize_name(entity.get("handle"))
|
|
305
|
+
if name:
|
|
306
|
+
return name
|
|
307
|
+
return ""
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _vcard_fn(vcard: object) -> str | None:
|
|
311
|
+
"""The first non-empty ``fn`` of a jCard ``["vcard", [[name, params, type, value]...]]``."""
|
|
312
|
+
if not isinstance(vcard, list) or len(vcard) < 2 or not isinstance(vcard[1], list):
|
|
313
|
+
return None
|
|
314
|
+
for prop in vcard[1]:
|
|
315
|
+
if isinstance(prop, list) and len(prop) >= 4 and prop[0] == "fn":
|
|
316
|
+
name = normalize_name(prop[3])
|
|
317
|
+
if name:
|
|
318
|
+
return name
|
|
319
|
+
return None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _error(domain: DomainHack, message: str, status: int | None = None) -> DomainCheckResult:
|
|
323
|
+
return DomainCheckResult(
|
|
324
|
+
domain=domain,
|
|
325
|
+
availability=Availability.ERROR,
|
|
326
|
+
raw_title=f"HTTP {status}" if status is not None else "",
|
|
327
|
+
error_message=message,
|
|
328
|
+
)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Pick the best availability source for a TLD.
|
|
2
|
+
|
|
3
|
+
Priority: RDAP (hard-coded override, then IANA bootstrap, then the bundled
|
|
4
|
+
bootstrap snapshot) -> WHOIS_SERVERS (TLDs without usable RDAP) ->
|
|
5
|
+
WHOIS_FALLBACK_SERVERS (RDAP TLDs; used only if RDAP resolution yields
|
|
6
|
+
nothing, with a stderr warning) -> unsupported (None).
|
|
7
|
+
|
|
8
|
+
Thanks to the bundled snapshot the choice does not depend on the network:
|
|
9
|
+
offline and online runs pick the same backend for every catalog TLD.
|
|
10
|
+
|
|
11
|
+
``.to`` is served by RDAP too (rdap.tonicregistry.to): it is the
|
|
12
|
+
registry's official protocol, it agreed with the Tonic web form on 23/23
|
|
13
|
+
test names (including registered names without NS records), and it gives
|
|
14
|
+
proper status codes and Retry-After, which a web form does not.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import functools
|
|
20
|
+
import sys
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from urllib.parse import urlsplit
|
|
23
|
+
|
|
24
|
+
from domainhack.adapters._circuit import HostCircuitBreaker
|
|
25
|
+
from domainhack.adapters.rdap_bootstrap import RdapBootstrap
|
|
26
|
+
from domainhack.adapters.rdap_registrar import RdapRegistrarClient
|
|
27
|
+
from domainhack.adapters.registry_sources import load_registry_sources
|
|
28
|
+
from domainhack.adapters.whois_registrar import (
|
|
29
|
+
WHOIS_FALLBACK_SERVERS,
|
|
30
|
+
WHOIS_SERVERS,
|
|
31
|
+
WhoisRegistrarClient,
|
|
32
|
+
)
|
|
33
|
+
from domainhack.domain.entities import TLD
|
|
34
|
+
from domainhack.ports.registrar import RegistrarClient
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@functools.cache
|
|
38
|
+
def _default_bootstrap() -> RdapBootstrap:
|
|
39
|
+
return RdapBootstrap()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def build_registrar_for(
|
|
43
|
+
tld: TLD,
|
|
44
|
+
delay: float,
|
|
45
|
+
timeout: float = 10.0,
|
|
46
|
+
breaker: HostCircuitBreaker | None = None,
|
|
47
|
+
contact: str | None = None,
|
|
48
|
+
bootstrap: RdapBootstrap | None = None,
|
|
49
|
+
) -> RegistrarClient | None:
|
|
50
|
+
"""Return a new client able to check domains under ``tld``, or None if unsupported.
|
|
51
|
+
|
|
52
|
+
Pass one ``breaker`` to every call of a run so clients that share a host
|
|
53
|
+
(e.g. .io/.sh/.ac/.me) also share its circuit. ``contact`` (an email) is
|
|
54
|
+
sent as the ``From`` header on RDAP requests. A multi-label ``tld``
|
|
55
|
+
(``com.ar``) is routed to the registry of its top-level domain (``ar``).
|
|
56
|
+
``bootstrap`` replaces the
|
|
57
|
+
process-wide default RDAP bootstrap (tests pass one with a fake fetcher).
|
|
58
|
+
"""
|
|
59
|
+
rdap = bootstrap if bootstrap is not None else _default_bootstrap()
|
|
60
|
+
base_url = rdap.base_url_for(tld.suffix)
|
|
61
|
+
if base_url is not None:
|
|
62
|
+
return RdapRegistrarClient(
|
|
63
|
+
base_url, delay=delay, timeout=timeout, breaker=breaker, contact=contact
|
|
64
|
+
)
|
|
65
|
+
# Multi-label suffixes (com.mx) are served by their top-level registry.
|
|
66
|
+
suffix = tld.top_level.lower()
|
|
67
|
+
if suffix in WHOIS_SERVERS:
|
|
68
|
+
return WhoisRegistrarClient(
|
|
69
|
+
delay=delay, timeout=timeout, servers=WHOIS_SERVERS, breaker=breaker
|
|
70
|
+
)
|
|
71
|
+
if suffix in WHOIS_FALLBACK_SERVERS:
|
|
72
|
+
host = WHOIS_FALLBACK_SERVERS[suffix].host
|
|
73
|
+
print(
|
|
74
|
+
f"warning: no RDAP server found for .{tld.suffix}; falling back to WHOIS ({host})",
|
|
75
|
+
file=sys.stderr,
|
|
76
|
+
)
|
|
77
|
+
return WhoisRegistrarClient(
|
|
78
|
+
delay=delay, timeout=timeout, servers=WHOIS_FALLBACK_SERVERS, breaker=breaker
|
|
79
|
+
)
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def supported_tlds(bootstrap: RdapBootstrap | None = None) -> set[str]:
|
|
84
|
+
"""All TLDs ``build_registrar_for`` can serve (may fetch the IANA bootstrap)."""
|
|
85
|
+
rdap = bootstrap if bootstrap is not None else _default_bootstrap()
|
|
86
|
+
return rdap.known_tlds() | set(WHOIS_SERVERS) | set(WHOIS_FALLBACK_SERVERS)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass(frozen=True)
|
|
90
|
+
class Backend:
|
|
91
|
+
"""How ``build_registrar_for`` would check a TLD, without creating a client.
|
|
92
|
+
|
|
93
|
+
``kind`` is ``"rdap"``, ``"whois"``, ``"whois-fallback"`` (RDAP TLD whose
|
|
94
|
+
RDAP server could not be resolved) or ``"unsupported"``. ``detail`` is the
|
|
95
|
+
date the source was verified, or the reason a TLD is unsupported.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
tld: str
|
|
99
|
+
kind: str
|
|
100
|
+
host: str
|
|
101
|
+
detail: str
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def describe_backend(suffix: str, bootstrap: RdapBootstrap | None = None) -> Backend:
|
|
105
|
+
"""The backend for ``suffix`` (``"cl"``, ``"com.ar"``), mirroring ``build_registrar_for``."""
|
|
106
|
+
rdap = bootstrap if bootstrap is not None else _default_bootstrap()
|
|
107
|
+
sources = load_registry_sources()
|
|
108
|
+
suffix = suffix.lower().lstrip(".")
|
|
109
|
+
top = suffix.rsplit(".", 1)[-1]
|
|
110
|
+
entry = sources.get(suffix) or sources.get(top)
|
|
111
|
+
verified = entry.verified_on if entry is not None and entry.verified_on else ""
|
|
112
|
+
base_url = rdap.base_url_for(suffix)
|
|
113
|
+
if base_url is not None:
|
|
114
|
+
detail = verified or "IANA RDAP bootstrap"
|
|
115
|
+
return Backend(suffix, "rdap", urlsplit(base_url).hostname or base_url, detail)
|
|
116
|
+
if top in WHOIS_SERVERS:
|
|
117
|
+
return Backend(suffix, "whois", WHOIS_SERVERS[top].host, verified)
|
|
118
|
+
if top in WHOIS_FALLBACK_SERVERS:
|
|
119
|
+
return Backend(suffix, "whois-fallback", WHOIS_FALLBACK_SERVERS[top].host, verified)
|
|
120
|
+
return Backend(suffix, "unsupported", "", sources.unsupported_reason(top) or "no source known")
|