ms-cloud-endpoints 0.1.0__py3-none-any.whl

ms_cloud_endpoints/output.py

@@ -0,0 +1,77 @@
+"""JSON and YAML serialisation of the collected document."""
+
+import json
+import os
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any, Literal, TextIO
+
+import yaml
+
+type Format = Literal["json", "yaml"]
+
+EXTENSIONS: dict[Format, str] = {"json": ".json", "yaml": ".yaml"}
+DEFAULT_BASENAME = "ms-cloud-endpoints"
+
+try:
+    _YamlDumper: type[yaml.SafeDumper] = yaml.CSafeDumper
+except AttributeError:  # PyYAML built without libyaml
+    _YamlDumper = yaml.SafeDumper
+
+
+def dump_json(document: dict[str, Any], stream: TextIO) -> None:
+    json.dump(document, stream, indent=2, ensure_ascii=False)
+    stream.write("\n")
+
+
+def dump_yaml(document: dict[str, Any], stream: TextIO) -> None:
+    yaml.dump(
+        document,
+        stream,
+        Dumper=_YamlDumper,
+        sort_keys=False,
+        allow_unicode=True,
+        default_flow_style=False,
+        width=120,
+    )
+
+
+DUMPERS = {"json": dump_json, "yaml": dump_yaml}
+
+
+def output_paths(output: str | Path, formats: list[Format]) -> dict[Format, Path]:
+    """Directory -> ``<dir>/ms-cloud-endpoints.<ext>``; file path -> that path (suffix replaced per format when
+    more than one format is written). A trailing separator marks a directory that does not exist yet."""
+    raw = os.fspath(output)
+    path = Path(raw)
+    if path.is_dir() or raw.endswith(("/", os.sep)):
+        return {fmt: path / f"{DEFAULT_BASENAME}{EXTENSIONS[fmt]}" for fmt in formats}
+    if len(formats) == 1:
+        return {formats[0]: path}
+    return {fmt: path.with_suffix(EXTENSIONS[fmt]) for fmt in formats}
+
+
+def _atomic_write(path: Path, fmt: Format, document: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
+    try:
+        os.fchmod(fd, 0o644)
+        with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream:
+            DUMPERS[fmt](document, stream)
+        os.replace(tmp_name, path)
+    except BaseException:
+        Path(tmp_name).unlink(missing_ok=True)
+        raise
+
+
+def write_document(document: dict[str, Any], formats: list[Format], output: str | Path | None) -> list[Path]:
+    if output is None:
+        if len(formats) != 1:
+            raise ValueError("writing more than one format requires --output")
+        DUMPERS[formats[0]](document, sys.stdout)
+        return []
+    paths = output_paths(output, formats)
+    for fmt, path in paths.items():
+        _atomic_write(path, fmt, document)
+    return list(paths.values())

ms_cloud_endpoints/sources/__init__.py

@@ -0,0 +1 @@
+"""Collectors for each documented Microsoft endpoint source."""

ms_cloud_endpoints/sources/m365.py

@@ -0,0 +1,68 @@
+"""Microsoft 365 IP Address and URL web service (https://endpoints.office.com)."""
+
+import logging
+from typing import Any
+from urllib.parse import quote, urlencode
+
+from ..fetch import Fetcher
+from ..model import SourceResult
+from ..netutil import normalise_domain, split_prefixes, utc_now
+
+log = logging.getLogger(__name__)
+
+BASE_URL = "https://endpoints.office.com"
+DOCS_URL = "https://learn.microsoft.com/en-us/microsoft-365/enterprise/microsoft-365-ip-web-service"
+
+
+def _url(method: str, instance: str, client_request_id: str) -> str:
+    return f"{BASE_URL}/{method}/{quote(instance)}?{urlencode({'ClientRequestId': client_request_id})}"
+
+
+def normalise_endpoint_set(record: dict[str, Any], *, include_ipv6: bool = True) -> dict[str, Any]:
+    ipv4, ipv6 = split_prefixes(record.get("ips", []))
+    return {
+        "id": record.get("id"),
+        "service_area": record.get("serviceArea"),
+        "service_area_display_name": record.get("serviceAreaDisplayName"),
+        "category": record.get("category"),
+        "required": record.get("required"),
+        "express_route": record.get("expressRoute"),
+        "urls": [normalise_domain(url) for url in record.get("urls", [])],
+        "ipv4": ipv4,
+        "ipv6": ipv6 if include_ipv6 else [],
+        "tcp_ports": record.get("tcpPorts"),
+        "udp_ports": record.get("udpPorts"),
+        "notes": record.get("notes"),
+    }
+
+
+def collect(
+    fetcher: Fetcher, instances: list[str], client_request_id: str, *, include_ipv6: bool = True
+) -> SourceResult:
+    data: dict[str, Any] = {}
+    sources: list[dict[str, Any]] = []
+    for instance in instances:
+        version_url = _url("version", instance, client_request_id)
+        endpoints_url = _url("endpoints", instance, client_request_id)
+        log.info("Fetching Microsoft 365 endpoints for %s", instance)
+        version = fetcher.get_json(version_url)
+        endpoint_sets = fetcher.get_json(endpoints_url)
+        if not isinstance(endpoint_sets, list):
+            raise ValueError(f"unexpected Microsoft 365 endpoints response for {instance}: {type(endpoint_sets)}")
+        latest = version.get("latest") if isinstance(version, dict) else None
+        data[instance] = {
+            "version": latest,
+            "endpoint_sets": [normalise_endpoint_set(record, include_ipv6=include_ipv6) for record in endpoint_sets],
+        }
+        sources.append(
+            {
+                "source": "m365",
+                "scope": instance,
+                "url": endpoints_url,
+                "documentation": DOCS_URL,
+                "version": latest,
+                "fetched_at": utc_now(),
+                "status": "ok",
+            }
+        )
+    return SourceResult(data, sources)

ms_cloud_endpoints/sources/m365_additional.py

@@ -0,0 +1,105 @@
+"""Heuristic parser for "Other endpoints not included in the Microsoft 365 IP Address and URL web service"."""
+
+import logging
+import re
+from typing import Any
+
+from ..fetch import Fetcher
+from ..html_tables import Node, find_column, parse_html, table_rows
+from ..model import SourceResult
+from ..netutil import extract_domains, extract_prefixes, utc_now
+
+log = logging.getLogger(__name__)
+
+URL = "https://learn.microsoft.com/en-us/microsoft-365/enterprise/additional-office365-ip-addresses-and-urls"
+
+# Labels used inside the Destination column to scope the following values to one cloud.
+CLOUD_LABELS: tuple[tuple[str, re.Pattern[str]], ...] = (
+    ("china", re.compile(r"^\s*(?:Microsoft Cloud China[^:]*|[^:]*21\s*Vianet[^:]*):", re.IGNORECASE)),
+    ("usgov", re.compile(r"^\s*(?:Microsoft Cloud for US Government|GCC High[^:]*|DoD[^:]*):", re.IGNORECASE)),
+    ("public", re.compile(r"^\s*(?:Public Cloud|Worldwide[^:]*):", re.IGNORECASE)),
+)
+CHINA_PURPOSE_RE = re.compile(r"21\s*Via\s*net|\bChina\b", re.IGNORECASE)
+
+
+def _segments(lines: list[str], default_cloud: str) -> dict[str, list[str]]:
+    """Group destination lines by the cloud label that precedes them."""
+    segments: dict[str, list[str]] = {}
+    cloud = default_cloud
+    for line in lines:
+        for label, pattern in CLOUD_LABELS:
+            if match := pattern.match(line):
+                cloud = label
+                line = line[match.end() :]
+                break
+        segments.setdefault(cloud, []).append(line)
+    return segments
+
+
+def _values(text: str) -> dict[str, list[str]]:
+    ipv4, ipv6 = extract_prefixes(text)
+    return {"domains": extract_domains(text), "ipv4": ipv4, "ipv6": ipv6}
+
+
+def parse_additional_endpoints(html: str) -> list[dict[str, Any]]:
+    root = parse_html(html)
+    for table in root.find_all("table"):
+        headers, rows = table_rows(table)
+        try:
+            row_col = find_column(headers, "row")
+            purpose_col = find_column(headers, "purpose")
+            destination_col = find_column(headers, "destination")
+        except LookupError:
+            continue
+        try:
+            type_col: int | None = find_column(headers, "type")
+        except LookupError:
+            type_col = None
+        try:
+            source_col: int | None = find_column(headers, "source")
+        except LookupError:
+            source_col = None
+
+        entries: list[dict[str, Any]] = []
+        for cells in rows:
+
+            def cell_lines(index: int | None, cells: list[Node] = cells) -> list[str]:
+                return cells[index].lines() if index is not None and index < len(cells) else []
+
+            purpose_lines = cell_lines(purpose_col)
+            destination_lines = cell_lines(destination_col)
+            purpose = "\n".join(purpose_lines)
+            default_cloud = "china" if CHINA_PURPOSE_RE.search(purpose) else "public"
+            # Some "Trusted Sites" rows only name their FQDNs in the purpose text.
+            scan_lines = destination_lines or purpose_lines
+            by_cloud = {
+                cloud: _values("\n".join(lines)) for cloud, lines in _segments(scan_lines, default_cloud).items()
+            }
+            row_text = cell_lines(row_col)
+            entries.append(
+                {
+                    "row": row_text[0] if row_text else None,
+                    "purpose": purpose,
+                    "source": "\n".join(cell_lines(source_col)) or None,
+                    "destination_text": "\n".join(destination_lines),
+                    "type": "\n".join(cell_lines(type_col)) or None,
+                    **_values("\n".join(scan_lines)),
+                    "by_cloud": by_cloud,
+                }
+            )
+        if entries:
+            return entries
+    raise LookupError("no table with Row/Purpose/Destination columns found on the additional endpoints page")
+
+
+def collect(fetcher: Fetcher, *, include_ipv6: bool = True) -> SourceResult:
+    log.info("Fetching Microsoft 365 additional endpoints page")
+    rows = parse_additional_endpoints(fetcher.get_text(URL))
+    if not include_ipv6:
+        for row in rows:
+            row["ipv6"] = []
+            for values in row["by_cloud"].values():
+                values["ipv6"] = []
+    source = {"source": "m365_additional", "scope": None, "url": URL, "version": None,
+              "fetched_at": utc_now(), "status": "ok"}  # fmt: skip
+    return SourceResult({"source_url": URL, "rows": rows}, [source])

ms_cloud_endpoints/sources/private_dns.py

@@ -0,0 +1,198 @@
+"""Azure Private Endpoint private DNS zone values and the Azure Backup geo-code mapping (Microsoft Learn pages)."""
+
+import itertools
+import logging
+import re
+import xml.etree.ElementTree as ET
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from ..fetch import Fetcher
+from ..html_tables import Node, find_column, parse_html, section_nodes, table_rows
+from ..model import SourceResult
+from ..netutil import PLACEHOLDER_RE, utc_now
+
+log = logging.getLogger(__name__)
+
+PRIVATE_DNS_URL = "https://learn.microsoft.com/en-us/azure/private-link/private-endpoint-dns"
+GEO_CODES_URL = "https://learn.microsoft.com/en-us/azure/backup/scripts/geo-code-list"
+
+TEMPLATE_RE = re.compile(r"(?:[A-Za-z0-9*-]|\{[A-Za-z]+\})+(?:\.(?:[A-Za-z0-9*-]|\{[A-Za-z]+\})+)*\.[A-Za-z]{2,63}")
+# Placeholders that can be expanded, mapped to the value list they are expanded from.
+REGION_PLACEHOLDERS = frozenset({"regionname", "region"})
+GEO_CODE_PLACEHOLDERS = frozenset({"regioncode"})
+SUP = frozenset({"sup"})
+
+
+@dataclass(frozen=True, slots=True)
+class PrivateDnsSection:
+    label: str
+    section_id: str
+    regions: Sequence[str] = ()
+
+
+def parse_geo_codes(html: str) -> list[dict[str, str]]:
+    """Extract ``GeoCodeRegionNameMap`` entries from the XML sample on the geo-code Learn page."""
+    root = parse_html(html)
+    for code in root.find_all("code") + root.find_all("pre"):
+        text = code.text(lines=True)
+        if "GeoCodeList" not in text:
+            continue
+        start = text.find("<GeoCodeList")
+        end = text.find("</GeoCodeList>")
+        if start < 0 or end < 0:
+            continue
+        xml_root = ET.fromstring(text[start : end + len("</GeoCodeList>")])
+        return [
+            {
+                "geo_code": element.attrib["GeoCode"],
+                "region_name": element.attrib["RegionName"],
+                "region": element.attrib["RegionName"].replace(" ", "").lower(),
+            }
+            for element in xml_root.iter("GeoCodeRegionNameMap")
+            if "GeoCode" in element.attrib and "RegionName" in element.attrib
+        ]
+    raise LookupError("GeoCodeList XML block not found on the geo-code page")
+
+
+def _footnote_texts(nodes: list[Node]) -> dict[str, str]:
+    """Footnotes are paragraphs that start with ``<sup>N</sup>``."""
+    footnotes: dict[str, str] = {}
+    for node in nodes:
+        if node.tag != "p":
+            continue
+        first = next((child for child in node.children if not (isinstance(child, str) and not child.strip())), None)
+        if isinstance(first, Node) and first.tag == "sup":
+            number = first.text()
+            text = node.text(skip=SUP)
+            if number and text:
+                footnotes[number] = text
+    return footnotes
+
+
+def _cell_values(cell: Node) -> list[str]:
+    values: list[str] = []
+    for line in cell.lines(skip=SUP):
+        values.extend(part for part in line.split() if part)
+    return values
+
+
+def _template_values(cell: Node) -> list[str]:
+    """Domain templates in a cell, lower-cased except for ``{placeholder}`` names; prose is dropped."""
+    templates: list[str] = []
+    for value in _cell_values(cell):
+        value = value.strip().rstrip(".,;")
+        if not TEMPLATE_RE.fullmatch(value):
+            continue
+        parts = PLACEHOLDER_RE.split(value)
+        # re.split with one group yields [text, name, text, name, ...]; rebuild with lower-cased text parts.
+        templates.append("".join(part.lower() if index % 2 == 0 else f"{{{part}}}" for index, part in enumerate(parts)))
+    return templates
+
+
+def placeholders(template: str) -> set[str]:
+    return {match.group("name") for match in PLACEHOLDER_RE.finditer(template)}
+
+
+def expand_template(template: str, values: Mapping[str, Sequence[str]]) -> tuple[list[str], set[str]]:
+    """Expand a zone template. Returns ``(concrete names, unresolved placeholder names)``."""
+    names = sorted(placeholders(template))
+    if not names:
+        return [template], set()
+    unresolved = {name for name in names if not values.get(name.lower())}
+    if unresolved:
+        return [], unresolved
+    expanded: list[str] = []
+    for combination in itertools.product(*(values[name.lower()] for name in names)):
+        result = template
+        for name, value in zip(names, combination, strict=True):
+            result = result.replace(f"{{{name}}}", value)
+        expanded.append(result)
+    return expanded, set()
+
+
+def _expand_all(templates: list[str], values: Mapping[str, Sequence[str]]) -> tuple[list[str], set[str]]:
+    expanded: dict[str, None] = {}
+    unresolved: set[str] = set()
+    for template in templates:
+        names, missing = expand_template(template, values)
+        expanded.update(dict.fromkeys(names))
+        unresolved |= missing
+    return list(expanded), unresolved
+
+
+def parse_private_dns_section(
+    html_root: Node, section: PrivateDnsSection, geo_codes: Sequence[str]
+) -> list[dict[str, Any]]:
+    nodes = section_nodes(html_root, section.section_id)
+    footnotes = _footnote_texts(nodes)
+    values: dict[str, Sequence[str]] = {}
+    if section.regions:
+        values |= dict.fromkeys(REGION_PLACEHOLDERS, list(section.regions))
+    if geo_codes:
+        values |= dict.fromkeys(GEO_CODE_PLACEHOLDERS, list(geo_codes))
+
+    entries: list[dict[str, Any]] = []
+    category: str | None = None
+    for node in nodes:
+        if node.tag in ("h3", "h4"):
+            category = node.text()
+            continue
+        if node.tag != "table":
+            continue
+        headers, rows = table_rows(node)
+        try:
+            resource_col = find_column(headers, "resource type")
+            subresource_col = find_column(headers, "subresource")
+            zone_col = find_column(headers, "private dns zone")
+            forwarder_col = find_column(headers, "forwarder")
+        except LookupError:
+            log.debug("Skipping table with headers %s in section %s", headers, section.section_id)
+            continue
+        for cells in rows:
+            if len(cells) <= max(resource_col, subresource_col, zone_col, forwarder_col):
+                continue
+            zone_templates = _template_values(cells[zone_col])
+            forwarder_templates = _template_values(cells[forwarder_col])
+            zones, unresolved_zones = _expand_all(zone_templates, values)
+            forwarders, unresolved_forwarders = _expand_all(forwarder_templates, values)
+            row_footnotes = sorted({sup.text() for cell in cells for sup in cell.find_all("sup")})
+            entries.append(
+                {
+                    "category": category,
+                    "resource_type": cells[resource_col].text(skip=SUP),
+                    "subresources": _cell_values(cells[subresource_col]),
+                    "zone_templates": zone_templates,
+                    "zones": zones,
+                    "forwarder_templates": forwarder_templates,
+                    "forwarders": forwarders,
+                    "unresolved_placeholders": sorted(unresolved_zones | unresolved_forwarders),
+                    "footnotes": [footnotes[number] for number in row_footnotes if number in footnotes],
+                }
+            )
+    if not entries:
+        raise LookupError(f"no private DNS zone tables found in section #{section.section_id}")
+    return entries
+
+
+def collect(fetcher: Fetcher, sections: list[PrivateDnsSection]) -> SourceResult:
+    sources: list[dict[str, Any]] = []
+    log.info("Fetching Azure Backup geo-codes")
+    geo_codes = parse_geo_codes(fetcher.get_text(GEO_CODES_URL))
+    sources.append(
+        {"source": "backup_geo_codes", "scope": None, "url": GEO_CODES_URL, "version": None,
+         "fetched_at": utc_now(), "status": "ok"}
+    )  # fmt: skip
+
+    log.info("Fetching Azure Private Endpoint DNS zone values")
+    html_root = parse_html(fetcher.get_text(PRIVATE_DNS_URL))
+    codes = [entry["geo_code"].lower() for entry in geo_codes]
+    data: dict[str, Any] = {"backup_geo_codes": geo_codes}
+    for section in sections:
+        data[section.label] = parse_private_dns_section(html_root, section, codes)
+        sources.append(
+            {"source": "private_dns_zones", "scope": section.label, "url": f"{PRIVATE_DNS_URL}#{section.section_id}",
+             "version": None, "fetched_at": utc_now(), "status": "ok"}
+        )  # fmt: skip
+    return SourceResult(data, sources)

ms_cloud_endpoints/sources/service_tags.py

@@ -0,0 +1,98 @@
+"""Azure IP Ranges and Service Tags, discovered from the Microsoft Download Center details pages."""
+
+import fnmatch
+import logging
+import re
+from typing import Any
+
+from ..fetch import Fetcher
+from ..model import SourceResult
+from ..netutil import split_prefixes, utc_now
+
+log = logging.getLogger(__name__)
+
+DETAILS_URL = "https://www.microsoft.com/en-us/download/details.aspx?id={download_id}"
+JSON_LINK_RE = re.compile(
+    r"https://download\.microsoft\.com/download/[^\"'\s<>]+?/ServiceTags_(?P<cloud>[A-Za-z]+)_(?P<date>\d{8})\.json"
+)
+
+
+def discover_json_url(details_html: str, cloud: str) -> str:
+    """Pick the newest ``ServiceTags_<cloud>_YYYYMMDD.json`` link from a download details page."""
+    candidates = [match for match in JSON_LINK_RE.finditer(details_html) if match.group("cloud") == cloud]
+    if not candidates:
+        raise LookupError(f"no ServiceTags_{cloud}_*.json link found on the download page")
+    return max(candidates, key=lambda match: match.group("date")).group(0)
+
+
+def tag_matches(name: str, patterns: list[str] | None) -> bool:
+    if not patterns:
+        return True
+    lowered = name.lower()
+    return any(fnmatch.fnmatchcase(lowered, pattern.lower()) for pattern in patterns)
+
+
+def normalise_tag(value: dict[str, Any], *, include_ipv6: bool = True) -> dict[str, Any]:
+    properties = value.get("properties", {})
+    ipv4, ipv6 = split_prefixes(properties.get("addressPrefixes", []))
+    return {
+        "name": value.get("name"),
+        "id": value.get("id"),
+        "region": properties.get("region") or None,
+        "region_id": properties.get("regionId"),
+        "system_service": properties.get("systemService") or None,
+        "platform": properties.get("platform"),
+        "network_features": properties.get("networkFeatures") or [],
+        "change_number": properties.get("changeNumber"),
+        "ipv4": ipv4,
+        "ipv6": ipv6 if include_ipv6 else [],
+    }
+
+
+def parse_service_tags(
+    document: dict[str, Any], *, tag_patterns: list[str] | None = None, include_ipv6: bool = True
+) -> dict[str, Any]:
+    values = document.get("values")
+    if not isinstance(values, list):
+        raise ValueError("service tag document has no 'values' list")
+    return {
+        "cloud": document.get("cloud"),
+        "change_number": document.get("changeNumber"),
+        "regions": sorted({region for value in values if (region := value.get("properties", {}).get("region"))}),
+        "tags": [
+            normalise_tag(value, include_ipv6=include_ipv6)
+            for value in values
+            if tag_matches(value.get("name", ""), tag_patterns)
+        ],
+    }
+
+
+def collect(
+    fetcher: Fetcher,
+    downloads: list[tuple[str, int]],
+    *,
+    tag_patterns: list[str] | None = None,
+    include_ipv6: bool = True,
+) -> SourceResult:
+    data: dict[str, Any] = {}
+    sources: list[dict[str, Any]] = []
+    for cloud, download_id in downloads:
+        details_url = DETAILS_URL.format(download_id=download_id)
+        log.info("Discovering Azure service tags download for %s", cloud)
+        json_url = discover_json_url(fetcher.get_text(details_url), cloud)
+        log.info("Fetching %s", json_url)
+        parsed = parse_service_tags(fetcher.get_json(json_url), tag_patterns=tag_patterns, include_ipv6=include_ipv6)
+        parsed["source_url"] = json_url
+        data[cloud] = parsed
+        sources.append(
+            {
+                "source": "azure_service_tags",
+                "scope": cloud,
+                "url": json_url,
+                "documentation": details_url,
+                "version": parsed["change_number"],
+                "fetched_at": utc_now(),
+                "status": "ok",
+            }
+        )
+    return SourceResult(data, sources)

ms_cloud_endpoints/summary.py

@@ -0,0 +1,82 @@
+"""Aggregate every collected source into deduplicated domain lists and collapsed IP prefix lists."""
+
+from collections.abc import Iterable
+from typing import Any
+
+from .netutil import PLACEHOLDER_RE, collapse, root_domain, wildcard_suffix
+
+
+class _Accumulator:
+    def __init__(self) -> None:
+        self.exact: set[str] = set()
+        self.wildcard: set[str] = set()
+        self.ipv4: list[str] = []
+        self.ipv6: list[str] = []
+
+    def add_domains(self, domains: Iterable[str]) -> None:
+        for domain in domains:
+            domain = domain.strip().lower()
+            if not domain:
+                continue
+            (self.wildcard if "*" in domain else self.exact).add(domain)
+
+    def add_templates(self, templates: Iterable[str]) -> None:
+        """Domain templates with ``{placeholder}`` labels are treated like wildcards."""
+        self.add_domains(PLACEHOLDER_RE.sub("*", template) for template in templates)
+
+    def add_prefixes(self, ipv4: Iterable[str], ipv6: Iterable[str]) -> None:
+        self.ipv4.extend(ipv4)
+        self.ipv6.extend(ipv6)
+
+
+def build_summary(document: dict[str, Any], clouds: list[str], *, include_ipv6: bool = True) -> dict[str, Any]:
+    acc = _Accumulator()
+
+    for instance in document.get("m365", {}).values():
+        for endpoint_set in instance.get("endpoint_sets", []):
+            acc.add_domains(endpoint_set.get("urls", []))
+            acc.add_prefixes(endpoint_set.get("ipv4", []), endpoint_set.get("ipv6", []))
+
+    for row in document.get("m365_additional", {}).get("rows", []):
+        for cloud, values in row.get("by_cloud", {}).items():
+            if cloud in clouds:
+                acc.add_domains(values.get("domains", []))
+                acc.add_prefixes(values.get("ipv4", []), values.get("ipv6", []))
+
+    for cloud in document.get("azure_service_tags", {}).values():
+        for tag in cloud.get("tags", []):
+            acc.add_prefixes(tag.get("ipv4", []), tag.get("ipv6", []))
+
+    for label, entries in document.get("private_dns_zones", {}).items():
+        if label == "backup_geo_codes":
+            continue
+        for entry in entries:
+            acc.add_domains(entry.get("zones", []))
+            acc.add_domains(entry.get("forwarders", []))
+            acc.add_templates(entry.get("zone_templates", []))
+            acc.add_templates(entry.get("forwarder_templates", []))
+
+    suffixes = {suffix for domain in acc.wildcard if (suffix := wildcard_suffix(domain))}
+    roots = {root_domain(domain) for domain in acc.exact | suffixes}
+    ipv4 = collapse(acc.ipv4)
+    ipv6 = collapse(acc.ipv6) if include_ipv6 else []
+    return {
+        "domains": {
+            "exact": sorted(acc.exact),
+            "wildcard": sorted(acc.wildcard),
+            "wildcard_suffixes": sorted(suffixes),
+            "root_domains": sorted(roots),
+        },
+        "ipv4": ipv4,
+        "ipv6": ipv6,
+        "counts": {
+            "exact_domains": len(acc.exact),
+            "wildcard_domains": len(acc.wildcard),
+            "wildcard_suffixes": len(suffixes),
+            "root_domains": len(roots),
+            "ipv4_prefixes_input": len(set(acc.ipv4)),
+            "ipv4_prefixes_collapsed": len(ipv4),
+            "ipv6_prefixes_input": len(set(acc.ipv6)) if include_ipv6 else 0,
+            "ipv6_prefixes_collapsed": len(ipv6),
+        },
+    }