ms-cloud-endpoints 0.1.0__py3-none-any.whl

ms_cloud_endpoints/__init__.py

@@ -0,0 +1,4 @@
+"""Collect Microsoft 365, Entra and Azure domains and IP ranges from the documented Microsoft sources."""
+
+__version__ = "0.1.0"
+GENERATOR = "ms-cloud-endpoints"

ms_cloud_endpoints/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

ms_cloud_endpoints/cli.py

@@ -0,0 +1,255 @@
+"""Command line entry point: fetch the selected sources and write the combined document as JSON and/or YAML."""
+
+import argparse
+import logging
+import os
+import sys
+import uuid
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from . import GENERATOR, __version__
+from .clouds import CLOUDS, Cloud, resolve_clouds
+from .fetch import Fetcher, HttpFetcher
+from .model import SourceResult
+from .netutil import utc_now
+from .output import Format, write_document
+from .sources import m365, m365_additional, private_dns, service_tags
+from .summary import build_summary
+
+log = logging.getLogger(GENERATOR)
+
+SOURCES = ("m365", "m365-additional", "service-tags", "private-dns")
+CLIENT_REQUEST_ID_ENV = "MS_ENDPOINTS_CLIENT_REQUEST_ID"
+
+EXIT_OK = 0
+EXIT_ERROR = 1
+EXIT_PARTIAL = 3
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=GENERATOR,
+        description=(
+            "Collect Microsoft 365, Entra and Azure domains and IP ranges from the documented Microsoft sources "
+            "and write them as JSON and/or YAML."
+        ),
+    )
+    parser.add_argument(
+        "--cloud",
+        action="append",
+        choices=[*CLOUDS, "all"],
+        help="cloud(s) to collect; repeatable (default: public)",
+    )
+    parser.add_argument(
+        "--source",
+        action="append",
+        choices=SOURCES,
+        help="source(s) to collect; repeatable (default: all)",
+    )
+    parser.add_argument(
+        "--tag",
+        action="append",
+        metavar="GLOB",
+        help="only include Azure service tags whose name matches GLOB (case-insensitive); repeatable, "
+        "e.g. --tag AzureCloud --tag 'AzureActiveDirectory*'",
+    )
+    parser.add_argument(
+        "--ipv6",
+        action=argparse.BooleanOptionalAction,
+        help="include IPv6 prefixes (--ipv6, the default) or drop them from the output (--no-ipv6)",
+    )
+    parser.add_argument("--no-summary", action="store_true", help="omit the aggregated summary section")
+    parser.add_argument(
+        "--format",
+        choices=["json", "yaml", "both"],
+        help="output format (default: json on stdout, both when --output is given)",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        metavar="PATH",
+        help="output file or directory (a trailing '/' or an existing directory receives "
+        "ms-cloud-endpoints.json/.yaml); default: stdout",
+    )
+    parser.add_argument(
+        "--client-request-id",
+        help=f"GUID for the Microsoft 365 web service (default: ${CLIENT_REQUEST_ID_ENV}, else a GUID persisted "
+        "per machine under $XDG_CONFIG_HOME/ms-cloud-endpoints/)",
+    )
+    parser.add_argument("--timeout", type=float, default=60.0, help="HTTP timeout in seconds (default: 60)")
+    parser.add_argument(
+        "--keep-going",
+        action="store_true",
+        help="record failed sources in metadata and still write the output (exit code 3)",
+    )
+    parser.add_argument("-v", "--verbose", action="count", default=0, help="more logging (-v info, -vv debug)")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    return parser
+
+
+def _config_dir() -> Path:
+    base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
+    return Path(base) / GENERATOR
+
+
+def resolve_client_request_id(explicit: str | None) -> str:
+    """Microsoft asks for one stable GUID per machine, so a generated GUID is persisted and reused."""
+    for candidate in (explicit, os.environ.get(CLIENT_REQUEST_ID_ENV)):
+        if candidate:
+            return str(uuid.UUID(candidate))
+    path = _config_dir() / "client-request-id"
+    try:
+        return str(uuid.UUID(path.read_text(encoding="utf-8").strip()))
+    except (OSError, ValueError):
+        pass
+    generated = str(uuid.uuid4())
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(generated + "\n", encoding="utf-8")
+    except OSError as exc:
+        log.warning("could not persist ClientRequestId to %s: %s", path, exc)
+    return generated
+
+
+def _resolve_formats(fmt: str | None, output: str | None) -> list[Format]:
+    if fmt is None:
+        fmt = "both" if output is not None else "json"
+    return ["json", "yaml"] if fmt == "both" else [fmt]  # type: ignore[list-item]
+
+
+def collect_document(
+    fetcher: Fetcher,
+    clouds: list[Cloud],
+    sources: list[str],
+    *,
+    client_request_id: str | None,
+    tag_patterns: list[str] | None,
+    include_ipv6: bool,
+    include_summary: bool,
+    keep_going: bool,
+) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    """Return ``(document, errors)``. Without ``keep_going`` the first failing source raises."""
+    source_records: list[dict[str, Any]] = []
+    errors: list[dict[str, Any]] = []
+    document: dict[str, Any] = {
+        "metadata": {
+            "generator": GENERATOR,
+            "generator_version": __version__,
+            "generated_at": None,
+            "clouds": [cloud.name for cloud in clouds],
+            "sources_selected": sources,
+            "options": {"tag_filter": tag_patterns or None, "ipv6": include_ipv6},
+            "client_request_id": client_request_id,
+            "sources": source_records,
+        }
+    }
+
+    def run(name: str, section: str, collector: Callable[[], SourceResult]) -> SourceResult | None:
+        try:
+            result = collector()
+        except Exception as exc:
+            if not keep_going:
+                raise
+            log.error("source %s failed: %s", name, exc)
+            error = {"source": name, "status": "error", "error": f"{type(exc).__name__}: {exc}"}
+            errors.append(error)
+            source_records.append(error)
+            return None
+        document[section] = result.data
+        source_records.extend(result.sources)
+        return result
+
+    if "m365" in sources and client_request_id is not None:
+        instances = [instance for cloud in clouds for instance in cloud.m365_instances]
+        run("m365", "m365", lambda: m365.collect(fetcher, instances, client_request_id, include_ipv6=include_ipv6))
+
+    if "m365-additional" in sources:
+        run(
+            "m365_additional",
+            "m365_additional",
+            lambda: m365_additional.collect(fetcher, include_ipv6=include_ipv6),
+        )
+
+    # Service tags run before the private DNS zones because their region lists expand {regionName} templates.
+    regions_by_cloud: dict[str, list[str]] = {}
+    if "service-tags" in sources:
+        downloads = [download for cloud in clouds for download in cloud.service_tag_downloads]
+        result = run(
+            "azure_service_tags",
+            "azure_service_tags",
+            lambda: service_tags.collect(fetcher, downloads, tag_patterns=tag_patterns, include_ipv6=include_ipv6),
+        )
+        if result is not None:
+            regions_by_cloud = {name: data.get("regions", []) for name, data in result.data.items()}
+
+    if "private-dns" in sources:
+        sections = [
+            private_dns.PrivateDnsSection(
+                label=cloud.private_dns_label,
+                section_id=cloud.private_dns_section,
+                regions=[
+                    region
+                    for tag_cloud, _ in cloud.service_tag_downloads
+                    for region in regions_by_cloud.get(tag_cloud, [])
+                ],
+            )
+            for cloud in clouds
+        ]
+        run("private_dns_zones", "private_dns_zones", lambda: private_dns.collect(fetcher, sections))
+
+    if include_summary:
+        document["summary"] = build_summary(document, [cloud.name for cloud in clouds], include_ipv6=include_ipv6)
+    return document, errors
+
+
+def main(argv: list[str] | None = None, *, fetcher: Fetcher | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    logging.basicConfig(
+        level=[logging.WARNING, logging.INFO, logging.DEBUG][min(args.verbose, 2)],
+        format="%(levelname)s %(name)s: %(message)s",
+        stream=sys.stderr,
+    )
+
+    formats = _resolve_formats(args.format, args.output)
+    if args.output is None and len(formats) > 1:
+        parser.error("--format both requires --output")
+    try:
+        clouds = resolve_clouds(args.cloud)
+    except ValueError as exc:
+        parser.error(str(exc))
+    sources = [source for source in SOURCES if source in (args.source or SOURCES)]
+    client_request_id: str | None = None
+    if "m365" in sources:
+        try:
+            client_request_id = resolve_client_request_id(args.client_request_id)
+        except ValueError as exc:
+            parser.error(f"invalid ClientRequestId (expected a GUID): {exc}")
+
+    try:
+        document, errors = collect_document(
+            fetcher or HttpFetcher(timeout=args.timeout),
+            clouds,
+            sources,
+            client_request_id=client_request_id,
+            tag_patterns=args.tag,
+            include_ipv6=args.ipv6 is not False,
+            include_summary=not args.no_summary,
+            keep_going=args.keep_going,
+        )
+    except Exception as exc:
+        log.error("%s: %s", type(exc).__name__, exc)
+        log.debug("traceback", exc_info=True)
+        return EXIT_ERROR
+    document["metadata"]["generated_at"] = utc_now()
+
+    try:
+        written = write_document(document, formats, args.output)
+    except (OSError, ValueError) as exc:
+        log.error("could not write output: %s", exc)
+        return EXIT_ERROR
+    for path in written:
+        print(f"wrote {path}", file=sys.stderr)
+    return EXIT_PARTIAL if errors else EXIT_OK

ms_cloud_endpoints/clouds.py

@@ -0,0 +1,51 @@
+"""Mapping of the user-facing cloud names to the identifiers used by each upstream source."""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class Cloud:
+    name: str
+    # Microsoft 365 IP Address and URL web service instances.
+    m365_instances: tuple[str, ...]
+    # Azure Service Tags: (cloud label used in the output, Microsoft Download Center details page id).
+    service_tag_downloads: tuple[tuple[str, int], ...]
+    # Section heading id on the "Azure Private Endpoint private DNS zone values" Learn page.
+    private_dns_section: str
+    private_dns_label: str
+
+
+CLOUDS: dict[str, Cloud] = {
+    "public": Cloud(
+        name="public",
+        m365_instances=("Worldwide",),
+        service_tag_downloads=(("Public", 56519),),
+        private_dns_section="commercial",
+        private_dns_label="Commercial",
+    ),
+    "china": Cloud(
+        name="china",
+        m365_instances=("China",),
+        service_tag_downloads=(("China", 57062),),
+        private_dns_section="china",
+        private_dns_label="China",
+    ),
+    "usgov": Cloud(
+        name="usgov",
+        m365_instances=("USGovDoD", "USGovGCCHigh"),
+        service_tag_downloads=(("AzureGovernment", 57063),),
+        private_dns_section="government",
+        private_dns_label="Government",
+    ),
+}
+
+
+def resolve_clouds(names: list[str] | None) -> list[Cloud]:
+    """Return the selected clouds in a stable order; ``all`` selects every cloud, default is ``public``."""
+    selected = set(names or ["public"])
+    if "all" in selected:
+        selected = set(CLOUDS)
+    unknown = selected - set(CLOUDS)
+    if unknown:
+        raise ValueError(f"unknown cloud(s): {', '.join(sorted(unknown))}")
+    return [cloud for key, cloud in CLOUDS.items() if key in selected]

ms_cloud_endpoints/fetch.py

@@ -0,0 +1,87 @@
+"""Minimal stdlib HTTP client with retries, used by every source module."""
+
+import gzip
+import json
+import logging
+import time
+import urllib.error
+import urllib.request
+import zlib
+from typing import Any, Protocol
+
+from . import GENERATOR, __version__
+
+log = logging.getLogger(__name__)
+
+# The Microsoft Download Center returns a reduced page to unknown clients, so present a browser-like agent.
+DEFAULT_USER_AGENT = (
+    f"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36 "
+    f"{GENERATOR}/{__version__}"
+)
+RETRY_STATUS = {429, 500, 502, 503, 504}
+
+
+class FetchError(RuntimeError):
+    pass
+
+
+class Fetcher(Protocol):
+    def get_text(self, url: str) -> str: ...
+
+    def get_json(self, url: str) -> Any: ...
+
+
+class HttpFetcher:
+    def __init__(
+        self,
+        *,
+        timeout: float = 60.0,
+        retries: int = 3,
+        backoff: float = 2.0,
+        user_agent: str = DEFAULT_USER_AGENT,
+    ) -> None:
+        self.timeout = timeout
+        self.retries = retries
+        self.backoff = backoff
+        self.user_agent = user_agent
+
+    def get_bytes(self, url: str) -> bytes:
+        request = urllib.request.Request(
+            url,
+            headers={
+                "User-Agent": self.user_agent,
+                "Accept-Encoding": "gzip, deflate",
+                "Accept-Language": "en-US,en;q=0.9",
+            },
+        )
+        last_error: Exception | None = None
+        for attempt in range(1, self.retries + 1):
+            try:
+                log.debug("GET %s (attempt %d)", url, attempt)
+                with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                    body = response.read()
+                    encoding = (response.headers.get("Content-Encoding") or "").lower()
+                if encoding == "gzip":
+                    body = gzip.decompress(body)
+                elif encoding == "deflate":
+                    body = zlib.decompress(body)
+                return body
+            except urllib.error.HTTPError as exc:
+                last_error = exc
+                if exc.code not in RETRY_STATUS:
+                    break
+            except (urllib.error.URLError, TimeoutError, ConnectionError) as exc:
+                last_error = exc
+            if attempt < self.retries:
+                time.sleep(self.backoff * attempt)
+        raise FetchError(f"failed to fetch {url}: {last_error}") from last_error
+
+    def get_text(self, url: str) -> str:
+        return self.get_bytes(url).decode("utf-8-sig")
+
+    def get_json(self, url: str) -> Any:
+        text = self.get_text(url)
+        try:
+            return json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise FetchError(f"invalid JSON from {url}: {exc}") from exc

ms_cloud_endpoints/html_tables.py

@@ -0,0 +1,160 @@
+"""Tiny DOM built on :mod:`html.parser`, with helpers for the Microsoft Learn page layout."""
+
+from collections.abc import Iterator
+from html.parser import HTMLParser
+
+VOID_TAGS = frozenset(
+    {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"}
+)
+# Elements whose text is rendered on its own line.
+BLOCK_TAGS = frozenset({"br", "p", "li", "div", "tr", "pre", "ul", "ol", "h1", "h2", "h3", "h4", "h5", "h6"})
+# Elements that never contribute readable text.
+SKIP_TAGS = frozenset({"script", "style", "template", "noscript"})
+
+
+class Node:
+    __slots__ = ("tag", "attrs", "children", "parent")
+
+    def __init__(self, tag: str, attrs: dict[str, str] | None = None, parent: "Node | None" = None) -> None:
+        self.tag = tag
+        self.attrs = attrs or {}
+        self.children: list[Node | str] = []
+        self.parent = parent
+
+    def __repr__(self) -> str:
+        return f"<Node {self.tag} {self.attrs.get('id', '')}>"
+
+    @property
+    def id(self) -> str | None:
+        return self.attrs.get("id")
+
+    def walk(self) -> Iterator["Node"]:
+        """Pre-order (document order) traversal of element nodes, including ``self``."""
+        stack: list[Node] = [self]
+        while stack:
+            node = stack.pop()
+            yield node
+            stack.extend(child for child in reversed(node.children) if isinstance(child, Node))
+
+    def find_all(self, tag: str) -> list["Node"]:
+        return [node for node in self.walk() if node.tag == tag]
+
+    def find_id(self, element_id: str) -> "Node | None":
+        return next((node for node in self.walk() if node.id == element_id), None)
+
+    def text(self, *, skip: frozenset[str] = frozenset(), lines: bool = False) -> str:
+        """Text content. ``skip`` drops whole elements (e.g. ``sup`` footnote markers); ``lines`` keeps block breaks."""
+        parts: list[str] = []
+        self._collect(parts, skip | SKIP_TAGS, lines)
+        raw = "".join(parts)
+        if lines:
+            return "\n".join(" ".join(line.split()) for line in raw.splitlines() if line.strip())
+        return " ".join(raw.split())
+
+    def lines(self, *, skip: frozenset[str] = frozenset()) -> list[str]:
+        text = self.text(skip=skip, lines=True)
+        return text.splitlines() if text else []
+
+    def _collect(self, parts: list[str], skip: frozenset[str], lines: bool) -> None:
+        for child in self.children:
+            if isinstance(child, str):
+                parts.append(child)
+                continue
+            if child.tag in skip:
+                continue
+            separator = ""
+            if child.tag in BLOCK_TAGS:
+                separator = "\n" if lines else " "
+            elif child.tag in ("td", "th"):
+                separator = " "
+            parts.append(separator)
+            child._collect(parts, skip, lines)
+            parts.append(separator)
+
+
+class _TreeBuilder(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.root = Node("#document")
+        self.current = self.root
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        node = Node(tag, {key: value or "" for key, value in attrs}, self.current)
+        self.current.children.append(node)
+        if tag not in VOID_TAGS:
+            self.current = node
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self.current.children.append(Node(tag, {key: value or "" for key, value in attrs}, self.current))
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in VOID_TAGS:
+            return
+        node: Node | None = self.current
+        while node is not None and node.tag != tag:
+            node = node.parent
+        # Ignore stray end tags; otherwise close everything up to and including the match.
+        if node is not None and node.parent is not None:
+            self.current = node.parent
+
+    def handle_data(self, data: str) -> None:
+        self.current.children.append(data)
+
+
+def parse_html(html: str) -> Node:
+    builder = _TreeBuilder()
+    builder.feed(html)
+    builder.close()
+    return builder.root
+
+
+def heading_level(node: Node) -> int | None:
+    if len(node.tag) == 2 and node.tag[0] == "h" and node.tag[1].isdigit():
+        return int(node.tag[1])
+    return None
+
+
+def section_nodes(root: Node, section_id: str) -> list[Node]:
+    """All element nodes in document order between the heading ``section_id`` and the next heading of equal or
+    higher rank."""
+    heading = root.find_id(section_id)
+    if heading is None or (level := heading_level(heading)) is None:
+        raise LookupError(f"section heading #{section_id} not found")
+    collected: list[Node] = []
+    started = False
+    for node in root.walk():
+        if node is heading:
+            started = True
+            continue
+        if not started:
+            continue
+        other_level = heading_level(node)
+        if other_level is not None and other_level <= level:
+            break
+        collected.append(node)
+    return collected
+
+
+def table_rows(table: Node) -> tuple[list[str], list[list[Node]]]:
+    """Return ``(header texts, body rows)``; header cells are taken from the first row that contains ``th``."""
+    headers: list[str] = []
+    rows: list[list[Node]] = []
+    for row in table.find_all("tr"):
+        cells = [child for child in row.children if isinstance(child, Node) and child.tag in ("th", "td")]
+        if not cells:
+            continue
+        if not headers and all(cell.tag == "th" for cell in cells):
+            headers = [cell.text(skip=frozenset({"sup"})) for cell in cells]
+            continue
+        rows.append(cells)
+    return headers, rows
+
+
+def find_column(headers: list[str], *candidates: str) -> int:
+    """Index of the first header that contains any of ``candidates`` (case-insensitive)."""
+    lowered = [header.lower() for header in headers]
+    for candidate in candidates:
+        for index, header in enumerate(lowered):
+            if candidate.lower() in header:
+                return index
+    raise LookupError(f"none of the columns {candidates!r} found in table headers {headers!r}")

ms_cloud_endpoints/model.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class SourceResult:
+    """Data collected from one source plus one metadata record per upstream document that was fetched."""
+
+    data: dict[str, Any]
+    sources: list[dict[str, Any]] = field(default_factory=list)

ms_cloud_endpoints/netutil.py

@@ -0,0 +1,100 @@
+"""IP prefix and domain name helpers shared by the source modules and the summary."""
+
+import ipaddress
+import re
+from collections.abc import Iterable
+from datetime import UTC, datetime
+
+type IPNetwork = ipaddress.IPv4Network | ipaddress.IPv6Network
+
+# FQDN or wildcard FQDN, optionally prefixed by a URL scheme and suffixed by a port or path. The TLD must be
+# alphabetic so that dotted IPv4 addresses are never taken for domains.
+DOMAIN_RE = re.compile(
+    r"(?<![\w.@/-])(?:https?://)?(?P<domain>(?:\*\.)?(?:[a-z0-9*](?:[a-z0-9*-]{0,61}[a-z0-9])?\.)+[a-z]{2,63})"
+    r"(?::\d+)?(?:/\S*)?(?![\w-])",
+    re.IGNORECASE,
+)
+# "{regionName}"-style placeholder labels in documented domain templates.
+PLACEHOLDER_RE = re.compile(r"\{(?P<name>[A-Za-z]+)\}")
+IPV4_RE = re.compile(r"(?<![\d.])\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?(?![\d.])")
+IPV6_RE = re.compile(r"(?<![\w:])(?:[0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}(?:/\d{1,3})?(?![\w:])", re.IGNORECASE)
+
+# Public suffixes with more than one label that occur in Microsoft endpoint lists; everything else is treated as a
+# single-label suffix. This avoids a dependency on the Public Suffix List.
+MULTI_LABEL_SUFFIXES = frozenset(
+    {
+        "co.uk", "org.uk", "gov.uk", "ac.uk", "com.cn", "net.cn", "gov.cn", "org.cn", "com.au", "net.au",
+        "co.jp", "ne.jp", "co.kr", "co.in", "co.nz", "com.br", "com.mx", "com.tw", "com.hk", "com.sg",
+        "co.za", "us.com",
+    }
+)  # fmt: skip
+
+
+def utc_now() -> str:
+    return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def parse_network(value: str) -> IPNetwork | None:
+    try:
+        return ipaddress.ip_network(value.strip(), strict=False)
+    except ValueError:
+        return None
+
+
+def split_prefixes(values: Iterable[str]) -> tuple[list[str], list[str]]:
+    """Normalise prefixes and split them by family, keeping the first-seen order and dropping duplicates."""
+    ipv4: dict[str, None] = {}
+    ipv6: dict[str, None] = {}
+    for value in values:
+        network = parse_network(value)
+        if network is None:
+            continue
+        (ipv4 if network.version == 4 else ipv6)[str(network)] = None
+    return list(ipv4), list(ipv6)
+
+
+def collapse(prefixes: Iterable[str]) -> list[str]:
+    networks = [network for value in prefixes if (network := parse_network(value)) is not None]
+    return [str(network) for network in ipaddress.collapse_addresses(networks)]
+
+
+def collapse_by_family(prefixes: Iterable[str]) -> tuple[list[str], list[str]]:
+    ipv4, ipv6 = split_prefixes(prefixes)
+    return collapse(ipv4), collapse(ipv6)
+
+
+def normalise_domain(value: str) -> str:
+    domain = value.strip().lower().rstrip(".")
+    for scheme in ("https://", "http://"):
+        domain = domain.removeprefix(scheme)
+    return domain.split("/", 1)[0].split(":", 1)[0]
+
+
+def wildcard_suffix(domain: str) -> str | None:
+    """``*.foo.com`` -> ``foo.com`` and ``autodiscover.*.onmicrosoft.com`` -> ``onmicrosoft.com``."""
+    if "*" not in domain:
+        return None
+    labels = domain.split(".")
+    last_wildcard = max(index for index, label in enumerate(labels) if "*" in label)
+    suffix = ".".join(labels[last_wildcard + 1 :])
+    return suffix or None
+
+
+def root_domain(domain: str) -> str:
+    labels = domain.lstrip("*.").split(".")
+    if len(labels) >= 3 and ".".join(labels[-2:]) in MULTI_LABEL_SUFFIXES:
+        return ".".join(labels[-3:])
+    return ".".join(labels[-2:])
+
+
+def extract_domains(text: str) -> list[str]:
+    found: dict[str, None] = {}
+    for match in DOMAIN_RE.finditer(text):
+        found[normalise_domain(match.group("domain"))] = None
+    return list(found)
+
+
+def extract_prefixes(text: str) -> tuple[list[str], list[str]]:
+    candidates = [match.group(0) for match in IPV4_RE.finditer(text)]
+    candidates += [match.group(0) for match in IPV6_RE.finditer(text) if match.group(0).count(":") >= 2]
+    return split_prefixes(candidates)