domainhack 0.2.0__py3-none-any.whl

domainhack/usecases/check_domains.py

@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Hashable, Iterable
+from dataclasses import dataclass, field
+
+from domainhack.domain.entities import Availability, DomainCheckResult, DomainHack
+from domainhack.ports.progress import NullProgressReporter, ProgressReporter
+from domainhack.ports.registrar import RegistrarClient
+from domainhack.ports.result_cache import ResultCache
+from domainhack.ports.result_writer import ResultWriter
+from domainhack.usecases.lanes import LaneOutcome, LaneScheduler
+
+LaneKey = Callable[[DomainHack], Hashable]
+"""Maps a domain to its lane: domains with equal keys are never checked concurrently.
+
+In production the key is the registry host serving the domain's TLD.
+"""
+
+DEFAULT_PARALLEL = 4
+# Domains waiting per lane. With a lazy source (range mode) at most
+# lanes x LANE_CAPACITY domains (plus their results) are held in memory.
+LANE_CAPACITY = 16
+# After Ctrl-C, how long in-flight checks may take to finish (and be written).
+SHUTDOWN_GRACE_SECONDS = 1.0
+
+
+def _one_lane(domain: DomainHack) -> Hashable:
+    return None
+
+
+@dataclass(frozen=True)
+class TldTally:
+    """How many domains under one TLD (suffix) were checked, and how many errored."""
+
+    suffix: str
+    checked: int = 0
+    errors: int = 0
+
+    @property
+    def unreachable(self) -> bool:
+        """Every check errored: the registry did not answer (timeouts, open circuit...)."""
+        return self.checked > 0 and self.errors == self.checked
+
+
+@dataclass(frozen=True)
+class CheckSummary:
+    """Outcome of a run: how many domains were checked and how they came out.
+
+    ``tlds`` breaks the checks down per TLD, in first-seen order.
+    """
+
+    available: int = 0
+    taken: int = 0
+    errors: int = 0
+    interrupted: bool = False
+    dropping: int = 0  # TAKEN names in redemption or pending delete (included in ``taken``)
+    # A breakdown of the counts above, so not part of equality.
+    tlds: tuple[TldTally, ...] = field(default=(), compare=False)
+
+    @property
+    def checked(self) -> int:
+        return self.available + self.taken + self.errors
+
+    @property
+    def tlds_with_errors(self) -> tuple[TldTally, ...]:
+        """TLDs with at least one ERROR, unreachable ones first (then first-seen order)."""
+        failed = [t for t in self.tlds if t.errors]
+        return tuple(sorted(failed, key=lambda t: not t.unreachable))
+
+
+@dataclass
+class _Tally:
+    counts: dict[Availability, int] = field(default_factory=lambda: dict.fromkeys(Availability, 0))
+    dropping: int = 0
+    # suffix -> [checked, errors], in first-seen order
+    by_tld: dict[str, list[int]] = field(default_factory=dict)
+
+    def add(self, result: DomainCheckResult) -> None:
+        self.counts[result.availability] += 1
+        self.dropping += result.is_dropping
+        tld = self.by_tld.setdefault(result.domain.tld.suffix, [0, 0])
+        tld[0] += 1
+        tld[1] += result.availability is Availability.ERROR
+
+    def summary(self, interrupted: bool) -> CheckSummary:
+        return CheckSummary(
+            available=self.counts[Availability.AVAILABLE],
+            taken=self.counts[Availability.TAKEN],
+            errors=self.counts[Availability.ERROR],
+            interrupted=interrupted,
+            dropping=self.dropping,
+            tlds=tuple(TldTally(s, c, e) for s, (c, e) in self.by_tld.items()),
+        )
+
+
+class CheckDomainsUseCase:
+    """Checks availability of domain hacks and writes results.
+
+    With ``parallel=1`` (the default) domains are checked one after another,
+    in input order. With ``parallel=N > 1`` they are checked by up to N
+    worker threads, one lane per ``lane_key`` (one registry host): domains
+    in the same lane are checked one at a time and in input order, different
+    lanes run concurrently. Results are then written in *completion* order,
+    unless ``keep_order`` buffers them to restore input order.
+
+    Only the thread calling ``execute`` touches the writer, the progress
+    reporter and the ``cache``. Cache hits are answered on that thread
+    before dispatch, so they never occupy a lane; live results are stored
+    there as they come back.
+    """
+
+    def __init__(
+        self,
+        registrar: RegistrarClient,
+        writer: ResultWriter,
+        progress: ProgressReporter | None = None,
+        *,
+        cache: ResultCache | None = None,
+        parallel: int = 1,
+        lane_key: LaneKey = _one_lane,
+        keep_order: bool = False,
+        lane_capacity: int = LANE_CAPACITY,
+        shutdown_grace: float | None = None,
+    ) -> None:
+        if parallel < 1:
+            raise ValueError("parallel must be at least 1")
+        self._registrar = registrar
+        self._writer = writer
+        self._progress = progress if progress is not None else NullProgressReporter()
+        self._cache = cache
+        self._parallel = parallel
+        self._lane_key = lane_key
+        self._keep_order = keep_order
+        self._lane_capacity = lane_capacity
+        self._shutdown_grace = (
+            shutdown_grace if shutdown_grace is not None else SHUTDOWN_GRACE_SECONDS
+        )
+
+    def execute(self, domains: Iterable[DomainHack], total: int | None = None) -> CheckSummary:
+        """Check every domain and return a summary.
+
+        A ``KeyboardInterrupt`` stops the run cleanly: the writers are flushed
+        (so partial output files are kept) and the summary has
+        ``interrupted=True``. In parallel mode checks already in flight get
+        ``shutdown_grace`` seconds to finish (and be written); the rest are
+        abandoned. Any other exception propagates after the flush. In
+        parallel mode an exception raised by a single check is reported as
+        an ERROR result for that domain instead.
+        """
+        tally = _Tally()
+        interrupted = False
+        try:
+            self._progress.start(total)
+            try:
+                if self._parallel == 1:
+                    self._run_sequential(domains, tally)
+                else:
+                    _ParallelRun(self, tally).run(domains)
+            except KeyboardInterrupt:
+                interrupted = True
+        finally:
+            # Close the bar first so anything printed afterwards appears below it.
+            try:
+                self._progress.close()
+            finally:
+                self._writer.flush()
+        return tally.summary(interrupted)
+
+    def _emit(self, result: DomainCheckResult, tally: _Tally) -> None:
+        self._writer.write_result(result)
+        tally.add(result)
+        self._progress.advance(result)
+
+    def _lookup(self, domain: DomainHack) -> DomainCheckResult | None:
+        return self._cache.lookup(domain) if self._cache is not None else None
+
+    def _store(self, result: DomainCheckResult) -> None:
+        if self._cache is not None:
+            self._cache.store(result)
+
+    def _run_sequential(self, domains: Iterable[DomainHack], tally: _Tally) -> None:
+        for domain in domains:
+            result = self._lookup(domain)
+            if result is None:
+                result = self._registrar.check_availability(domain)
+                self._store(result)
+            self._emit(result, tally)
+
+
+class _ParallelRun:
+    """One parallel ``execute``: feeds the lanes and consumes their results.
+
+    Runs entirely on the calling thread; the workers live in ``LaneScheduler``.
+    """
+
+    def __init__(self, use_case: CheckDomainsUseCase, tally: _Tally) -> None:
+        self._uc = use_case
+        self._tally = tally
+        self._scheduler = LaneScheduler(
+            use_case._registrar.check_availability,
+            workers=use_case._parallel,
+            lane_capacity=use_case._lane_capacity,
+        )
+        self._outstanding = 0  # submitted, outcome not consumed yet
+        # keep_order: results waiting for an earlier one, by input index.
+        self._held: dict[int, DomainCheckResult] = {}
+        self._next_index = 0
+        # Max results held for keep_order before feeding pauses (memory bound).
+        self._max_held = max(use_case._lane_capacity * use_case._parallel, 1) * 4
+
+    def run(self, domains: Iterable[DomainHack]) -> None:
+        try:
+            for index, domain in enumerate(domains):
+                self._drain(block=False)
+                cached = self._uc._lookup(domain)
+                if cached is not None:
+                    self._deliver(index, cached)
+                else:
+                    self._submit(index, domain)
+                # keep_order: hold at most ~max_held results behind a slow check.
+                while len(self._held) > self._max_held:
+                    self._drain(block=True)
+            self._scheduler.close()
+            while self._outstanding:
+                self._drain(block=True)
+        except KeyboardInterrupt:
+            # Stop dispatching; keep whatever finishes within the grace period.
+            self._scheduler.stop(self._uc._shutdown_grace)
+            self._drain_finished()
+            self._release_held()
+            raise
+        finally:
+            self._scheduler.stop()
+
+    def _submit(self, index: int, domain: DomainHack) -> None:
+        lane = self._uc._lane_key(domain)
+        while not self._scheduler.try_submit(lane, index, domain):
+            # That lane is full: handle a result (from any lane), then retry.
+            # The full lane is busy or next in line, so a result always comes.
+            self._drain(block=True)
+        self._outstanding += 1
+
+    def _drain(self, *, block: bool) -> None:
+        """Handle one outcome when ``block``; otherwise every outcome already available."""
+        if block:
+            outcome = self._scheduler.next_outcome()
+            assert outcome is not None
+            self._handle(outcome)
+            return
+        while self._outstanding:
+            outcome = self._scheduler.next_outcome(timeout=0)
+            if outcome is None:
+                return
+            self._handle(outcome)
+
+    def _drain_finished(self) -> None:
+        """After an interrupt: write results that are ready, ignore failures."""
+        while self._outstanding:
+            outcome = self._scheduler.next_outcome(timeout=0)
+            if outcome is None:
+                return
+            self._outstanding -= 1
+            if outcome.result is not None:
+                self._uc._store(outcome.result)
+                self._deliver(outcome.index, outcome.result)
+
+    def _handle(self, outcome: LaneOutcome) -> None:
+        self._outstanding -= 1
+        if outcome.fatal is not None:
+            raise outcome.fatal
+        assert outcome.result is not None
+        self._uc._store(outcome.result)
+        self._deliver(outcome.index, outcome.result)
+
+    def _deliver(self, index: int, result: DomainCheckResult) -> None:
+        if not self._uc._keep_order:
+            self._uc._emit(result, self._tally)
+            return
+        self._held[index] = result
+        while self._next_index in self._held:
+            self._uc._emit(self._held.pop(self._next_index), self._tally)
+            self._next_index += 1
+
+    def _release_held(self) -> None:
+        """Write results still held for ordering (after a gap left by an interrupt)."""
+        for index in sorted(self._held):
+            self._uc._emit(self._held.pop(index), self._tally)

domainhack/usecases/confirm_dns.py

@@ -0,0 +1,64 @@
+"""Second opinion from public DNS on registry answers.
+
+The registry (RDAP/WHOIS) stays the source of truth; DNS only adds
+evidence. An AVAILABLE name that is delegated in DNS is a conflict (see
+``DomainCheckResult.dns_conflict``): something is wrong (a stale or broken
+registry answer, a reserved name...), so it must not be reported as free.
+A TAKEN name without NS records is registered but not in use.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from collections.abc import Iterable
+from concurrent.futures import ThreadPoolExecutor
+
+from domainhack.domain.entities import Availability, DnsEvidence, DomainCheckResult
+from domainhack.domain.parking import parking_hint_for
+from domainhack.ports.dns_lookup import DnsLookup
+
+DEFAULT_DNS_WORKERS = 8
+
+_CONFIRMED = frozenset({Availability.AVAILABLE, Availability.TAKEN})
+
+
+class ConfirmWithDns:
+    """Fills ``DomainCheckResult.dns`` for AVAILABLE and TAKEN results.
+
+    ERROR results pass through unchanged (there is no registry answer to
+    confirm). Lookups run on a pool of at most ``workers`` threads, so the
+    ``lookup`` must be thread-safe.
+    """
+
+    def __init__(self, lookup: DnsLookup, workers: int = DEFAULT_DNS_WORKERS) -> None:
+        if workers < 1:
+            raise ValueError("workers must be at least 1")
+        self._lookup = lookup
+        self.workers = workers
+
+    @staticmethod
+    def applies_to(result: DomainCheckResult) -> bool:
+        """True for results whose registry answer DNS can confirm (AVAILABLE, TAKEN)."""
+        return result.availability in _CONFIRMED
+
+    def confirm(self, result: DomainCheckResult) -> DomainCheckResult:
+        """A copy of ``result`` with ``dns`` filled; ``result`` itself if it is an ERROR."""
+        if not self.applies_to(result):
+            return result
+        try:
+            evidence = self._lookup.lookup(result.domain.fqdn)
+        except Exception as exc:  # the port says never; a broken adapter must not end the run
+            evidence = DnsEvidence(error=f"DNS lookup failed: {exc or type(exc).__name__}")
+        parked = result.parked_hint
+        if not parked and result.availability is Availability.TAKEN and evidence.is_delegated:
+            # The registry gave no nameservers (e.g. WHOIS TLDs): classify DNS's instead.
+            parked = parking_hint_for(evidence.nameservers)
+        return dataclasses.replace(result, dns=evidence, parked_hint=parked)
+
+    def apply(self, results: Iterable[DomainCheckResult]) -> list[DomainCheckResult]:
+        """Confirm every result, concurrently; the output keeps the input order."""
+        items = list(results)
+        if not any(self.applies_to(r) for r in items):
+            return items
+        with ThreadPoolExecutor(max_workers=self.workers, thread_name_prefix="dns-confirm") as pool:
+            return list(pool.map(self.confirm, items))

domainhack/usecases/estimate_run.py

@@ -0,0 +1,80 @@
+"""Estimate how many queries a run sends and how long it takes, per host."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+
+from domainhack.domain.entities import TLD
+
+# Above this many queries a brute-force run needs an explicit --yes: registry
+# terms of use ask clients not to bulk-query, and 10,000 queries is ~3 h at 1 query/s.
+GUARDRAIL_MAX_QUERIES = 10_000
+
+
+@dataclass(frozen=True)
+class Pacing:
+    """Where a TLD's queries go and the minimum spacing between them there."""
+
+    host: str
+    interval: float
+
+
+@dataclass(frozen=True)
+class HostLoad:
+    host: str
+    queries: int
+    interval: float
+
+    @property
+    def seconds(self) -> float:
+        return self.queries * self.interval
+
+
+@dataclass(frozen=True)
+class RunEstimate:
+    hosts: tuple[HostLoad, ...]
+
+    @property
+    def queries(self) -> int:
+        return sum(h.queries for h in self.hosts)
+
+    @property
+    def seconds(self) -> float:
+        """Lower bound on the run time: the busiest host's queries x its interval.
+
+        Hosts are paced independently, so requests to different hosts do not
+        wait for each other; the slowest host sets the pace. Network latency
+        and adaptive slow-downs only add to this.
+        """
+        return max((h.seconds for h in self.hosts), default=0.0)
+
+    def exceeds(self, threshold: int = GUARDRAIL_MAX_QUERIES) -> bool:
+        return self.queries > threshold
+
+
+def estimate_run(queries_by_tld: Mapping[TLD, int], pacing: Callable[[TLD], Pacing]) -> RunEstimate:
+    """Group per-TLD query counts by host (TLDs can share one, e.g. .io/.sh)."""
+    loads: dict[str, tuple[int, float]] = {}
+    for tld, queries in queries_by_tld.items():
+        p = pacing(tld)
+        count, interval = loads.get(p.host, (0, 0.0))
+        loads[p.host] = (count + queries, max(interval, p.interval))
+    return RunEstimate(
+        tuple(HostLoad(host, count, interval) for host, (count, interval) in loads.items())
+    )
+
+
+def format_duration(seconds: float) -> str:
+    """``"45 s"``, ``"12 min"``, ``"5 h 4 min"``, ``"5 d 12 h"``."""
+    total = round(seconds)
+    if total < 60:
+        return f"{total} s"
+    minutes, _ = divmod(total, 60)
+    if minutes < 60:
+        return f"{minutes} min"
+    hours, minutes = divmod(minutes, 60)
+    if hours < 24:
+        return f"{hours} h {minutes} min"
+    days, hours = divmod(hours, 24)
+    return f"{days} d {hours} h"

domainhack/usecases/filter_words.py

@@ -0,0 +1,37 @@
+from collections.abc import Iterator, Sequence
+
+from domainhack.domain.entities import TLD, DomainHack, InvalidLabelError
+from domainhack.ports.word_source import WordSource
+
+
+class FilterWordsUseCase:
+    """Filters words from a source that form valid domain hacks for one or more TLDs.
+
+    A word may match several TLDs (e.g. "testing" -> "testi.ng" and "test.ing");
+    one DomainHack is yielded per match, in word order and then in TLD order.
+
+    Matches whose SLD is not a valid label for that TLD (``can't`` -> ``can't.to``,
+    an IDN under a TLD without IDN support, too short for ``.it``...) are not
+    yielded; they are counted in ``skipped`` so callers can report them.
+    """
+
+    def __init__(
+        self, word_source: WordSource, tld: TLD | Sequence[TLD], min_length: int = 0
+    ) -> None:
+        self._word_source = word_source
+        self._tlds: tuple[TLD, ...] = (tld,) if isinstance(tld, TLD) else tuple(tld)
+        self._min_length = min_length
+        self.skipped = 0
+
+    def execute(self) -> Iterator[DomainHack]:
+        for word in self._word_source.words():
+            if self._min_length and len(word) < self._min_length:
+                continue
+            for tld in self._tlds:
+                try:
+                    hack = DomainHack.from_word(word, tld)
+                except InvalidLabelError:
+                    self.skipped += 1
+                    continue
+                if hack is not None:
+                    yield hack

domainhack/usecases/generate_range.py

@@ -0,0 +1,107 @@
+import itertools
+import string
+from collections.abc import Iterator, Sequence
+
+from domainhack.domain.entities import TLD, DomainHack, InvalidLabelError
+from domainhack.domain.label_rules import label_rule_for
+from domainhack.ports.word_source import WordSource
+
+MAX_RANGE_LENGTH = 6
+_ALPHABET = string.ascii_lowercase
+_BASE = len(_ALPHABET)
+
+
+class RangeWordSource(WordSource):
+    """Generates all lowercase letter combinations up to a given length."""
+
+    def __init__(self, max_length: int, end_at: str | None = None) -> None:
+        if max_length < 1:
+            raise ValueError("max_length must be >= 1")
+        if max_length > MAX_RANGE_LENGTH:
+            raise ValueError(f"max_length must be <= {MAX_RANGE_LENGTH}")
+        self._max_length = max_length
+        self._end_at = end_at
+
+    def words(self) -> Iterator[str]:
+        for length in range(1, self._max_length + 1):
+            for combo in itertools.product(_ALPHABET, repeat=length):
+                word = "".join(combo)
+                yield word
+                if self._end_at and word == self._end_at:
+                    return
+
+    def total(self) -> int:
+        """Exact number of words ``words()`` will yield, without generating them.
+
+        If ``end_at`` can never be produced (wrong length, non-lowercase chars), the
+        generator runs to exhaustion, so the full count is returned.
+        """
+        return self.count()
+
+    def count(self, min_length: int = 1, prefix: str = "") -> int:
+        """How many words ``words()`` yields with ``len >= min_length`` and ``prefix``.
+
+        Computed arithmetically: words come out shortest first and, within one
+        length, in lexicographic order, so ``end_at`` cuts at a known point.
+        """
+        end = self._end_at
+        if not end or len(end) > self._max_length or any(c not in _ALPHABET for c in end):
+            end = None
+        count = 0
+        for length in range(max(min_length, len(prefix), 1), self._max_length + 1):
+            free = length - len(prefix)
+            if end is None or length < len(end):
+                count += _BASE**free
+            elif length == len(end):
+                head = end[: len(prefix)]
+                if head > prefix:
+                    count += _BASE**free
+                elif head == prefix:
+                    offset = 0
+                    for char in end[len(prefix) :]:
+                        offset = offset * _BASE + _ALPHABET.index(char)
+                    count += offset + 1
+            else:
+                break
+        return count
+
+
+class RangeCandidatesUseCase:
+    """Brute-force candidates: every generated SLD under every TLD, SLD-major.
+
+    Labels the TLD's registry would refuse (see ``label_rules``, e.g. names
+    shorter than 3 characters under ``.it``) are skipped and counted in
+    ``skipped`` instead of being yielded, so they never reach a registrar.
+    """
+
+    def __init__(self, source: RangeWordSource, tlds: TLD | Sequence[TLD]) -> None:
+        self._source = source
+        self._tlds: tuple[TLD, ...] = (tlds,) if isinstance(tlds, TLD) else tuple(tlds)
+        self.skipped = 0
+
+    def execute(self) -> Iterator[DomainHack]:
+        for sld in self._source.words():
+            for tld in self._tlds:
+                try:
+                    yield DomainHack.from_sld(sld, tld)
+                except InvalidLabelError:
+                    self.skipped += 1
+
+    def total(self) -> int:
+        """Exact number of candidates ``execute()`` yields."""
+        return sum(self.totals_by_tld().values())
+
+    def totals_by_tld(self) -> dict[TLD, int]:
+        """Exact number of candidates ``execute()`` yields for each TLD.
+
+        Generated SLDs are 1-6 lowercase ASCII letters, so only the rules'
+        ``min_length`` and ``forbidden_prefixes`` can reject them.
+        """
+        totals: dict[TLD, int] = {}
+        for tld in self._tlds:
+            rule = label_rule_for(tld.suffix)
+            count = self._source.count(min_length=rule.min_length)
+            for prefix in rule.forbidden_prefixes:
+                count -= self._source.count(min_length=rule.min_length, prefix=prefix)
+            totals[tld] = count
+        return totals