domainhack 0.2.0__py3-none-any.whl

domainhack/adapters/csv_writer.py

@@ -0,0 +1,79 @@
+import csv
+
+from domainhack.adapters._registration import format_utc
+from domainhack.adapters._text_sink import TextSink, TextTarget
+from domainhack.domain.entities import DomainCheckResult
+from domainhack.ports.result_writer import ResultWriter
+
+# New columns are appended at the end so existing column positions stay stable.
+CSV_FIELDS = (
+    "fqdn",
+    "display",
+    "word",
+    "sld",
+    "tld",
+    "availability",
+    "error_message",
+    "statuses",
+    "expires_at",
+    "registered_at",
+    "registrar",
+    "nameservers",
+    "parked_hint",
+    "dns_nameservers",
+    "dns_conflict",
+)
+STATUS_SEPARATOR = ";"
+LIST_SEPARATOR = STATUS_SEPARATOR
+
+
+class CsvResultWriter(ResultWriter):
+    """Streams domain check results to a CSV file, one row per result.
+
+    ``fqdn`` is the ASCII name that was queried (A-label for IDNs); ``display``
+    is the name as written (U-label), and equals ``fqdn`` for ASCII names.
+    ``statuses`` is the registry status list joined with ``;`` and
+    ``expires_at`` an ISO 8601 UTC date-time; both are empty when unknown.
+    ``registered_at`` (ISO 8601 UTC), ``registrar``, ``nameservers`` (joined
+    with ``;``) and ``parked_hint`` (the parking/aftermarket service the
+    nameservers point to, e.g. ``domainrecover``) are registry details of a
+    taken name, empty when unknown. ``dns_nameservers`` (joined with ``;``)
+    are the NS hosts public DNS returned, empty when DNS was not consulted;
+    ``dns_conflict`` is ``true`` when the registry said available but DNS
+    shows a delegation, else ``false``.
+
+    The header is written immediately and each row is flushed as it is written,
+    so an interrupted run keeps every result checked so far.
+    """
+
+    def __init__(self, target: TextTarget) -> None:
+        self._sink = TextSink(target)
+        self._writer = csv.writer(self._sink.stream)
+        self._writer.writerow(CSV_FIELDS)
+        self._sink.stream.flush()
+
+    def write_result(self, result: DomainCheckResult) -> None:
+        domain = result.domain
+        self._writer.writerow(
+            (
+                domain.fqdn,
+                domain.display,
+                domain.word,
+                domain.sld,
+                domain.tld.suffix,
+                result.availability.value,
+                result.error_message,
+                STATUS_SEPARATOR.join(result.statuses),
+                format_utc(result.expires_at),
+                format_utc(result.registered_at),
+                result.registrar,
+                LIST_SEPARATOR.join(result.nameservers),
+                result.parked_hint,
+                LIST_SEPARATOR.join(result.dns.nameservers) if result.dns is not None else "",
+                "true" if result.dns_conflict else "false",
+            )
+        )
+        self._sink.stream.flush()
+
+    def flush(self) -> None:
+        self._sink.close()

domainhack/adapters/dns_confirming_writer.py

@@ -0,0 +1,72 @@
+"""A ResultWriter decorator that adds DNS evidence before results are written."""
+
+from __future__ import annotations
+
+from collections import deque
+from concurrent.futures import Future, ThreadPoolExecutor
+
+from domainhack.domain.entities import DomainCheckResult
+from domainhack.ports.result_writer import ResultWriter
+from domainhack.usecases.confirm_dns import ConfirmWithDns
+
+
+class DnsConfirmingWriter(ResultWriter):
+    """Confirms each result with DNS (``ConfirmWithDns``), then passes it on.
+
+    Results stream: each lookup starts as soon as its result arrives, on a
+    pool of ``confirm.workers`` threads, and results are handed to ``inner``
+    in arrival order as soon as every earlier one is done. ``inner`` is only
+    called from the thread that calls ``write_result``/``flush`` (the
+    check's main thread), never from the pool.
+
+    ``flush`` waits for the outstanding lookups. If that wait is interrupted
+    (Ctrl-C), the results still pending are written without DNS evidence
+    and ``inner`` is flushed anyway, so partial output files are kept.
+    ``conflicts`` counts results written with ``dns_conflict``.
+    """
+
+    def __init__(self, inner: ResultWriter, confirm: ConfirmWithDns) -> None:
+        self._inner = inner
+        self._confirm = confirm
+        self._pool = ThreadPoolExecutor(
+            max_workers=confirm.workers, thread_name_prefix="dns-confirm"
+        )
+        self._pending: deque[tuple[DomainCheckResult, Future[DomainCheckResult] | None]] = deque()
+        self.conflicts = 0
+
+    def write_result(self, result: DomainCheckResult) -> None:
+        future = (
+            self._pool.submit(self._confirm.confirm, result)
+            if self._confirm.applies_to(result)
+            else None
+        )
+        self._pending.append((result, future))
+        self._emit_ready()
+
+    def flush(self) -> None:
+        try:
+            while self._pending:
+                original, future = self._pending[0]
+                confirmed = future.result() if future is not None else original
+                self._pending.popleft()
+                self._emit(confirmed)
+        finally:
+            # Interrupted: keep what was checked, without the missing evidence.
+            while self._pending:
+                original, future = self._pending.popleft()
+                done = future is not None and future.done() and future.exception() is None
+                self._emit(future.result() if done and future is not None else original)
+            self._pool.shutdown(wait=False, cancel_futures=True)
+            self._inner.flush()
+
+    def _emit_ready(self) -> None:
+        while self._pending:
+            original, future = self._pending[0]
+            if future is not None and not future.done():
+                return
+            self._pending.popleft()
+            self._emit(future.result() if future is not None else original)
+
+    def _emit(self, result: DomainCheckResult) -> None:
+        self.conflicts += result.dns_conflict
+        self._inner.write_result(result)

domainhack/adapters/dns_resolver.py

@@ -0,0 +1,120 @@
+"""Public DNS evidence for a name, via dnspython (a second opinion, never the verdict).
+
+``DnsPythonLookup.lookup("sumanda.com")`` asks the configured recursive
+resolver (the system's by default) for the name's NS records and, when the
+name exists, whether it has an A or AAAA record:
+
+- NXDOMAIN: the name does not exist in DNS -> empty evidence, no error.
+- NS present: the name is delegated, so it is certainly registered. NS hosts
+  are lowercase, without the trailing dot, sorted.
+- Timeout, SERVFAIL (every server failed) or any other failure of the NS
+  query -> ``error`` is set and the other fields mean nothing.
+
+A failed A/AAAA query after a successful NS query only leaves
+``has_address`` False: the delegation, the part that matters, is known.
+``lookup`` never raises.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable, Iterable
+from typing import Any, Protocol
+
+import dns.exception
+import dns.resolver
+
+from domainhack.domain.entities import DnsEvidence
+from domainhack.ports.dns_lookup import DnsLookup
+
+DEFAULT_DNS_TIMEOUT = 3.0
+
+
+class Resolver(Protocol):
+    """The part of ``dns.resolver.Resolver`` this adapter uses (tests pass a fake)."""
+
+    def resolve(
+        self,
+        qname: str,
+        rdtype: str,
+        *,
+        raise_on_no_answer: bool,
+        lifetime: float | None,
+        search: bool | None,
+    ) -> Iterable[Any]: ...
+
+
+class DnsPythonLookup(DnsLookup):
+    """``DnsLookup`` backed by dnspython.
+
+    ``resolver`` is used as given (shared by every thread); by default each
+    thread gets its own ``dns.resolver.Resolver()`` built from the system
+    configuration, so parallel callers never share mutable resolver state.
+    ``timeout`` bounds each query (NS, A, AAAA) in seconds.
+    """
+
+    def __init__(
+        self,
+        resolver: Resolver | None = None,
+        timeout: float = DEFAULT_DNS_TIMEOUT,
+        resolver_factory: Callable[[], Resolver] = dns.resolver.Resolver,
+    ) -> None:
+        self._shared = resolver
+        self._factory = resolver_factory
+        self._timeout = timeout
+        self._local = threading.local()
+
+    def _resolver(self) -> Resolver:
+        if self._shared is not None:
+            return self._shared
+        resolver: Resolver | None = getattr(self._local, "resolver", None)
+        if resolver is None:
+            resolver = self._factory()
+            self._local.resolver = resolver
+        return resolver
+
+    def lookup(self, fqdn: str) -> DnsEvidence:
+        name = fqdn.strip().rstrip(".").lower() + "."
+        try:
+            resolver = self._resolver()
+            records = self._query(resolver, name, "NS")
+        except dns.resolver.NXDOMAIN:
+            return DnsEvidence()
+        except Exception as exc:  # never raise: DNS is only a second opinion
+            return DnsEvidence(error=describe_dns_error(exc))
+        hosts = (_host(record) for record in records)
+        nameservers = tuple(sorted({host for host in hosts if host}))
+        return DnsEvidence(nameservers=nameservers, has_address=self._has_address(resolver, name))
+
+    def _has_address(self, resolver: Resolver, name: str) -> bool:
+        for rdtype in ("A", "AAAA"):
+            try:
+                if any(True for _ in self._query(resolver, name, rdtype)):
+                    return True
+            except Exception:  # NXDOMAIN, timeout...: no address known
+                return False
+        return False
+
+    def _query(self, resolver: Resolver, name: str, rdtype: str) -> list[Any]:
+        answer = resolver.resolve(
+            name, rdtype, raise_on_no_answer=False, lifetime=self._timeout, search=False
+        )
+        return list(answer)
+
+
+def _host(record: Any) -> str:
+    """An NS record's target, lowercase, without the trailing dot."""
+    target = getattr(record, "target", record)
+    return str(target).strip().rstrip(".").lower()
+
+
+def describe_dns_error(exc: BaseException) -> str:
+    """A short, stable description of a failed DNS query."""
+    if isinstance(exc, dns.exception.Timeout):
+        return "DNS timeout"
+    if isinstance(exc, dns.resolver.NoNameservers):
+        return "DNS SERVFAIL (no nameserver answered)"
+    if isinstance(exc, dns.resolver.NoResolverConfiguration):
+        return "DNS not configured (no resolver found)"
+    detail = str(exc) or type(exc).__name__
+    return f"DNS error: {detail}"

domainhack/adapters/file_word_source.py

@@ -0,0 +1,51 @@
+from collections.abc import Iterator
+from pathlib import Path
+
+from domainhack.ports.word_source import WordSource
+
+DEFAULT_ENCODING = "utf-8"
+
+
+class WordListError(Exception):
+    """The word list cannot be read; the message is meant for the end user."""
+
+
+class FileWordSource(WordSource):
+    """Reads words line-by-line from a text file.
+
+    Decoding is strict: a byte that is not valid in ``encoding`` raises
+    ``WordListError`` instead of being replaced, because a mis-decoded word
+    (e.g. ``ma\\ufffdana``) would turn into a wrong domain to query.
+    """
+
+    def __init__(self, file_path: Path, encoding: str = DEFAULT_ENCODING) -> None:
+        self._file_path = file_path
+        self._encoding = encoding
+
+    def check_readable(self) -> None:
+        """Fail fast (``WordListError``) if the file cannot be opened at all."""
+        try:
+            with open(self._file_path, "rb"):
+                pass
+        except OSError as exc:
+            raise self._os_error(exc) from exc
+
+    def _os_error(self, exc: OSError) -> WordListError:
+        reason = exc.strerror or str(exc)
+        return WordListError(f"cannot read word list '{self._file_path}': {reason}")
+
+    def words(self) -> Iterator[str]:
+        try:
+            with open(self._file_path, encoding=self._encoding) as f:
+                for line in f:
+                    word = line.strip()
+                    if word:
+                        yield word
+        except UnicodeDecodeError as exc:
+            bad = exc.object[exc.start]
+            raise WordListError(
+                f"cannot read word list '{self._file_path}': not valid {self._encoding} "
+                f"(byte {bad:#04x}); convert it to UTF-8 or pass --encoding (e.g. latin-1)"
+            ) from exc
+        except OSError as exc:
+            raise self._os_error(exc) from exc

domainhack/adapters/iana_tlds.py

@@ -0,0 +1,110 @@
+"""The public suffixes a name can be registered under, from bundled package data.
+
+Two files ship with the package:
+
+- ``data/iana_tlds.txt``: a verbatim snapshot of IANA's
+  https://data.iana.org/TLD/tlds-alpha-by-domain.txt (its first line, a
+  ``# Version ...`` comment, says when it was taken). IDN TLDs appear in
+  their ``xn--`` form and count as known.
+- ``data/second_level.json``: a curated list of common registrable
+  second-level suffixes (``com.ar``, ``co.uk``...), each checked against the
+  ICANN section of the Public Suffix List (the file cites the version).
+
+The list is snapshot-only on purpose: IANA adds TLDs a few times a year at
+most, and a fixed list keeps hack suggestions deterministic and offline.
+Refresh it by downloading the IANA file over ``data/iana_tlds.txt``
+unchanged. Whether a suffix can actually be *checked* is a different
+question, answered by ``registrar_catalog.supported_tlds``.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from importlib import resources
+
+from domainhack.ports.known_tlds import KnownTlds
+
+IANA_SNAPSHOT = "data/iana_tlds.txt"
+SECOND_LEVEL_FILE = "data/second_level.json"
+
+
+def _read_data(name: str) -> str:
+    return resources.files("domainhack").joinpath(name).read_text(encoding="utf-8")
+
+
+def parse_iana_tlds(text: str) -> tuple[str, list[str]]:
+    """Parse the IANA file into its ``# Version ...`` header (without ``#``) and lowercase TLDs."""
+    version = ""
+    tlds: list[str] = []
+    for raw in text.splitlines():
+        line = raw.strip()
+        if not line:
+            continue
+        if line.startswith("#"):
+            if not version:
+                version = line.lstrip("#").strip()
+            continue
+        tlds.append(line.lower())
+    return version, tlds
+
+
+def load_iana_snapshot() -> tuple[str, list[str]]:
+    """The bundled IANA list: (version header, lowercase TLDs)."""
+    return parse_iana_tlds(_read_data(IANA_SNAPSHOT))
+
+
+def load_second_level() -> list[str]:
+    """The curated second-level suffixes, lowercase, in file order (LATAM first)."""
+    data = json.loads(_read_data(SECOND_LEVEL_FILE))
+    return [s.lower() for s in data["suffixes"]]
+
+
+class IanaTldList(KnownTlds):
+    """Known public suffixes: every IANA TLD plus curated second-level suffixes.
+
+    ``tlds`` and ``second_level`` replace the bundled files (tests pass small
+    lists). Matching is case-insensitive and ignores a leading dot.
+    """
+
+    def __init__(
+        self,
+        tlds: Iterable[str] | None = None,
+        second_level: Iterable[str] | None = None,
+    ) -> None:
+        if tlds is None:
+            self.version, tld_list = load_iana_snapshot()
+        else:
+            self.version, tld_list = "", list(tlds)
+        levels = load_second_level() if second_level is None else list(second_level)
+        self._tlds = frozenset(_normalize(t) for t in tld_list)
+        self._second_level = tuple(dict.fromkeys(_normalize(s) for s in levels))
+        self._all = self._tlds | frozenset(self._second_level)
+        # "comar" -> ["com.ar"]: suffixes as they appear at the end of a word.
+        self._by_joined: dict[str, list[str]] = {}
+        for suffix in sorted(self._all):
+            self._by_joined.setdefault(suffix.replace(".", ""), []).append(suffix)
+
+    @property
+    def top_level_domains(self) -> frozenset[str]:
+        return self._tlds
+
+    @property
+    def second_level_suffixes(self) -> tuple[str, ...]:
+        """The curated second-level suffixes, in file order."""
+        return self._second_level
+
+    def is_known(self, suffix: str) -> bool:
+        return _normalize(suffix) in self._all
+
+    def suffixes_of(self, name: str) -> list[str]:
+        word = name.strip().lower()
+        found: list[str] = []
+        # Longest ending first; position 0 would leave an empty SLD.
+        for start in range(1, len(word)):
+            found.extend(self._by_joined.get(word[start:], ()))
+        return found
+
+
+def _normalize(suffix: str) -> str:
+    return suffix.strip().lower().lstrip(".")

domainhack/adapters/json_writer.py

@@ -0,0 +1,51 @@
+import json
+
+from domainhack.adapters._registration import format_utc
+from domainhack.adapters._text_sink import TextSink, TextTarget
+from domainhack.domain.entities import DomainCheckResult
+from domainhack.ports.result_writer import ResultWriter
+
+
+class JsonResultWriter(ResultWriter):
+    """Streams domain check results as JSON Lines (one JSON object per line).
+
+    Each line is flushed as it is written, so an interrupted run leaves a valid
+    file containing every result checked so far. ``fqdn`` is the queried ASCII
+    name (A-label for IDNs) and ``display`` the name as written. ``statuses``
+    is a list of registry statuses and ``expires_at`` an ISO 8601 UTC
+    date-time or null. ``registered_at`` (ISO 8601 UTC or null),
+    ``registrar``, ``nameservers`` (a list) and ``parked_hint`` (the
+    parking/aftermarket service, e.g. ``"domainrecover"``) are registry
+    details of a taken name, empty when unknown. ``dns_nameservers`` is the
+    list of NS hosts public DNS returned (empty when DNS was not consulted)
+    and ``dns_conflict`` is true when the registry said available but DNS
+    shows a delegation.
+    """
+
+    def __init__(self, target: TextTarget) -> None:
+        self._sink = TextSink(target)
+
+    def write_result(self, result: DomainCheckResult) -> None:
+        domain = result.domain
+        record = {
+            "fqdn": domain.fqdn,
+            "display": domain.display,
+            "word": domain.word,
+            "sld": domain.sld,
+            "tld": domain.tld.suffix,
+            "availability": result.availability.value,
+            "error_message": result.error_message,
+            "statuses": list(result.statuses),
+            "expires_at": format_utc(result.expires_at) or None,
+            "registered_at": format_utc(result.registered_at) or None,
+            "registrar": result.registrar,
+            "nameservers": list(result.nameservers),
+            "parked_hint": result.parked_hint,
+            "dns_nameservers": list(result.dns.nameservers) if result.dns is not None else [],
+            "dns_conflict": result.dns_conflict,
+        }
+        self._sink.stream.write(json.dumps(record, ensure_ascii=False) + "\n")
+        self._sink.stream.flush()
+
+    def flush(self) -> None:
+        self._sink.close()

domainhack/adapters/pacing.py

@@ -0,0 +1,46 @@
+"""Where a registrar client sends a TLD's queries, and how fast (for run estimates)."""
+
+from __future__ import annotations
+
+from collections.abc import Hashable
+
+import httpx
+
+from domainhack.adapters.rdap_registrar import RdapRegistrarClient
+from domainhack.adapters.whois_registrar import WhoisRegistrarClient
+from domainhack.domain.entities import TLD
+from domainhack.ports.registrar import RegistrarClient
+from domainhack.usecases.estimate_run import Pacing
+
+
+def pacing_for(client: RegistrarClient | None, tld: TLD, delay: float) -> Pacing:
+    """The host serving ``tld`` through ``client`` and its base query interval.
+
+    RDAP clients are paced at ``delay``; WHOIS servers at max(``delay``, the
+    server's ``min_interval``). An unknown client is assumed to be on a host
+    of its own, paced at ``delay``.
+    """
+    if isinstance(client, RdapRegistrarClient):
+        return Pacing(httpx.URL(client.base_url).host, delay)
+    if isinstance(client, WhoisRegistrarClient):
+        server = client.server_for(tld.suffix)
+        if server is not None:
+            return Pacing(server.host, max(delay, server.min_interval))
+    return Pacing(f".{tld.suffix}", delay)
+
+
+def lane_for(client: RegistrarClient | None, tld: TLD) -> Hashable:
+    """The parallel-check lane for ``tld``: domains in one lane never run concurrently.
+
+    RDAP and WHOIS clients get the registry host they query, so TLDs served
+    by the same host (.io/.sh/.ac/.me) share a lane and that host never has
+    more than one request in flight. Any other client gets a lane of its own
+    per instance: nothing is known about its host or its thread-safety, so
+    it is never called concurrently. Unsupported TLDs (no client) share a
+    lane; the router answers them without network access.
+    """
+    if isinstance(client, (RdapRegistrarClient, WhoisRegistrarClient)):
+        pacing = pacing_for(client, tld, 0.0)
+        if not pacing.host.startswith("."):
+            return pacing.host
+    return ("client", id(client)) if client is not None else ("unsupported",)