ms-split-router 0.1.0__py3-none-any.whl

ms_split_router/data.py

@@ -0,0 +1,152 @@
+"""Load the ms-cloud-endpoints document from a file, or fetch it by running the ms-cloud-endpoints command."""
+
+import json
+import logging
+import shutil
+import subprocess
+from collections.abc import Callable, Sequence
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .config import Config
+
+log = logging.getLogger(__name__)
+
+# Labels used inside the collector document -> the collector's --cloud value.
+SERVICE_TAG_CLOUDS = {"public": "public", "china": "china", "azuregovernment": "usgov"}
+M365_INSTANCES = {"worldwide": "public", "china": "china", "usgovdod": "usgov", "usgovgcchigh": "usgov"}
+PRIVATE_DNS_CLOUDS = {"commercial": "public", "china": "china", "government": "usgov"}
+
+COLLECTOR_EXIT_PARTIAL = 3
+
+type Runner = Callable[[Sequence[str]], subprocess.CompletedProcess[str]]
+
+
+class DataError(RuntimeError):
+    """The collector document could not be loaded or fetched."""
+
+
+class PartialDataError(DataError):
+    """The collector reported that one or more sources failed (exit code 3)."""
+
+    def __init__(self, message: str, document: dict[str, Any]) -> None:
+        super().__init__(message)
+        self.document = document
+
+
+def _validate(document: Any, origin: str) -> dict[str, Any]:
+    if not isinstance(document, dict) or "metadata" not in document:
+        raise DataError(f"{origin} is not an ms-cloud-endpoints document (no 'metadata' key)")
+    if not any(key in document for key in ("m365", "azure_service_tags", "m365_additional", "private_dns_zones")):
+        raise DataError(f"{origin} contains no endpoint data")
+    return document
+
+
+def load_document(path: Path) -> dict[str, Any]:
+    """Read a collector JSON or YAML file (format chosen by suffix, else by content)."""
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise DataError(f"cannot read {path}: {exc.strerror or exc}") from None
+    try:
+        if path.suffix.lower() == ".json" or text.lstrip().startswith("{"):
+            document = json.loads(text)
+        else:
+            loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
+            document = yaml.load(text, Loader=loader)
+    except (json.JSONDecodeError, yaml.YAMLError) as exc:
+        raise DataError(f"{path}: cannot parse: {exc}") from None
+    return _validate(document, str(path))
+
+
+def fetch_arguments(config: Config) -> list[str]:
+    """Collector arguments that fetch only what the configured links need."""
+    clouds: set[str] = set()
+    sources: set[str] = set()
+    tags: set[str] = set()
+    ipv6 = False
+    for link in config.split_links:
+        routes, domains = link.routes, link.domains
+        if routes.azure:
+            sources.add("service-tags")
+            tags.update(routes.azure.service_tags)
+            clouds.update(SERVICE_TAG_CLOUDS.get(cloud.lower(), "public") for cloud in routes.azure.clouds)
+        if routes.m365 or "m365" in domains.derive_from:
+            sources.add("m365")
+            instances = routes.m365.instances if routes.m365 else ("Worldwide",)
+            clouds.update(M365_INSTANCES.get(instance.lower(), "public") for instance in instances)
+        if routes.m365_additional or "m365_additional" in domains.derive_from:
+            sources.add("m365-additional")
+            if routes.m365_additional:
+                clouds.update(routes.m365_additional.clouds)
+        if "private_dns" in domains.derive_from:
+            sources.add("private-dns")
+            clouds.update(PRIVATE_DNS_CLOUDS.get(cloud.lower(), "public") for cloud in domains.private_dns_clouds)
+        if routes.ipv6 is not False:
+            ipv6 = True
+
+    args = ["--format", "json", "--no-summary"]
+    for cloud in sorted(clouds or {"public"}):
+        args += ["--cloud", cloud]
+    for source in sorted(sources):
+        args += ["--source", source]
+    for tag in sorted(tags):
+        args += ["--tag", tag]
+    if not ipv6:
+        args.append("--no-ipv6")
+    return args
+
+
+def _run(command: Sequence[str]) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(command, capture_output=True, text=True, check=False)
+
+
+def fetch_document(config: Config, *, runner: Runner = _run) -> dict[str, Any]:
+    """Run the collector and parse its JSON from stdout. Raises ``PartialDataError`` on collector exit code 3."""
+    executable = config.data.fetch_command
+    if runner is _run and shutil.which(executable) is None:
+        raise DataError(
+            f"{executable!r} not found on PATH; install ms-cloud-endpoints (pipx install ./ms-cloud-endpoints) "
+            "or set data.file / --data"
+        )
+    command = [executable, *fetch_arguments(config), *config.data.fetch_args]
+    log.info("Running %s", " ".join(command))
+    result = runner(command)
+    if result.stderr:
+        for line in result.stderr.splitlines():
+            log.info("collector: %s", line)
+    if result.returncode not in (0, COLLECTOR_EXIT_PARTIAL):
+        raise DataError(f"{executable} exited with code {result.returncode}: {result.stderr.strip()[-500:]}")
+    try:
+        document = _validate(json.loads(result.stdout), f"{executable} output")
+    except json.JSONDecodeError as exc:
+        raise DataError(f"{executable} did not print JSON: {exc}") from None
+    if result.returncode == COLLECTOR_EXIT_PARTIAL:
+        failed = [
+            f"{record.get('source')}/{record.get('scope')}"
+            for record in document.get("metadata", {}).get("sources", [])
+            if record.get("status") != "ok"
+        ]
+        raise PartialDataError(f"collector reported failed source(s): {', '.join(failed) or 'unknown'}", document)
+    return document
+
+
+def data_versions(document: dict[str, Any]) -> list[str]:
+    """Human-readable upstream versions recorded in the document, for file headers."""
+    versions = [
+        f"Microsoft 365 {name} {data.get('version')}" for name, data in sorted(document.get("m365", {}).items())
+    ]
+    versions += [
+        f"Service Tags {name} change {data.get('change_number')}"
+        for name, data in sorted(document.get("azure_service_tags", {}).items())
+    ]
+    return versions
+
+
+def collected_ipv6(document: dict[str, Any]) -> bool:
+    """False only when the collector recorded that it dropped IPv6 prefixes (``ms-cloud-endpoints --no-ipv6``)."""
+    metadata = document.get("metadata")
+    options = metadata.get("options") if isinstance(metadata, dict) else None
+    return not (isinstance(options, dict) and options.get("ipv6") is False)

ms_split_router/discover.py

@@ -0,0 +1,146 @@
+"""Discover mirrored links and their gateways from ``ip -json`` output."""
+
+import json
+import logging
+import subprocess
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from .config import Match
+from .netutil import IPAddress, parse_address
+
+log = logging.getLogger(__name__)
+
+type IpRunner = Callable[[Sequence[str]], str]
+
+
+class DiscoveryError(RuntimeError):
+    """A link or gateway could not be discovered."""
+
+
+@dataclass(frozen=True, slots=True)
+class Gateway:
+    address: IPAddress
+    onlink: bool
+    metric: int
+
+
+@dataclass(frozen=True, slots=True)
+class LinkInfo:
+    ifname: str
+    mac: str | None
+    altnames: tuple[str, ...]
+    operstate: str
+    gateway4: Gateway | None = None
+    gateway6: Gateway | None = None
+    addresses: tuple[str, ...] = field(default_factory=tuple)
+
+
+def run_ip(args: Sequence[str]) -> str:
+    try:
+        result = subprocess.run(["ip", "-json", *args], capture_output=True, text=True, check=False)
+    except FileNotFoundError:
+        raise DiscoveryError("the 'ip' command (iproute2) is not installed") from None
+    if result.returncode != 0:
+        raise DiscoveryError(f"ip -json {' '.join(args)} failed: {result.stderr.strip()}")
+    return result.stdout
+
+
+def _json(runner: IpRunner, args: Sequence[str]) -> list[dict[str, Any]]:
+    output = runner(args).strip()
+    if not output:
+        return []
+    try:
+        value = json.loads(output)
+    except json.JSONDecodeError as exc:
+        raise DiscoveryError(f"cannot parse ip -json {' '.join(args)}: {exc}") from None
+    return value if isinstance(value, list) else []
+
+
+def _default_gateway(routes: list[dict[str, Any]], version: int) -> Gateway | None:
+    candidates = []
+    for route in routes:
+        if route.get("dst") != "default" or "gateway" not in route:
+            continue
+        try:
+            address = parse_address(route["gateway"])
+        except ValueError:
+            continue
+        if address.version == version:
+            flags = route.get("flags", [])
+            candidates.append(Gateway(address, "onlink" in flags, int(route.get("metric", 0))))
+    return min(candidates, key=lambda gateway: gateway.metric, default=None)
+
+
+class Discovery:
+    """Lazily queries ``ip -json`` so that nothing runs unless auto-detection is needed."""
+
+    def __init__(self, runner: IpRunner | None = None) -> None:
+        self._runner = runner or run_ip
+        self._links: list[dict[str, Any]] | None = None
+        self._cache: dict[str, LinkInfo] = {}
+
+    def links(self) -> list[dict[str, Any]]:
+        if self._links is None:
+            self._links = [link for link in _json(self._runner, ["link", "show"]) if link.get("ifname") != "lo"]
+        return self._links
+
+    def resolve(self, match: Match) -> str:
+        """Kernel interface name of the link selected by ``match``."""
+        found = []
+        for link in self.links():
+            if match.mac and str(link.get("address", "")).lower() == match.mac:
+                found.append(link["ifname"])
+            elif match.altname and match.altname in link.get("altnames", []):
+                found.append(link["ifname"])
+            elif match.name and match.name in (link.get("ifname"), *link.get("altnames", [])):
+                found.append(link["ifname"])
+        if not found:
+            available = ", ".join(f"{link['ifname']} ({link.get('address')})" for link in self.links())
+            raise DiscoveryError(f"no link with {match.describe()} (available: {available or 'none'})")
+        if len(found) > 1:
+            raise DiscoveryError(f"{match.describe()} matches several links: {', '.join(found)}")
+        return found[0]
+
+    def info(self, match: Match) -> LinkInfo:
+        ifname = self.resolve(match)
+        if ifname not in self._cache:
+            link = next(link for link in self.links() if link["ifname"] == ifname)
+            routes4 = _json(self._runner, ["-4", "route", "show", "default", "dev", ifname])
+            routes6 = _json(self._runner, ["-6", "route", "show", "default", "dev", ifname])
+            addresses = [
+                f"{address['local']}/{address['prefixlen']}"
+                for entry in _json(self._runner, ["address", "show", "dev", ifname])
+                for address in entry.get("addr_info", [])
+                if address.get("scope") == "global"
+            ]
+            for route in (*routes4, *routes6):
+                route.setdefault("dst", "default")
+            self._cache[ifname] = LinkInfo(
+                ifname=ifname,
+                mac=link.get("address"),
+                altnames=tuple(link.get("altnames", [])),
+                operstate=str(link.get("operstate", "UNKNOWN")),
+                gateway4=_default_gateway(routes4, 4),
+                gateway6=_default_gateway(routes6, 6),
+                addresses=tuple(addresses),
+            )
+        return self._cache[ifname]
+
+
+def describe_links(discovery: Discovery) -> list[str]:
+    """One line per link, for ``--show-links``."""
+    lines = []
+    for link in discovery.links():
+        info = discovery.info(Match(name=link["ifname"]))
+        gateways = [
+            f"gw{gateway.address.version}={gateway.address}{' onlink' if gateway.onlink else ''}"
+            for gateway in (info.gateway4, info.gateway6)
+            if gateway
+        ]
+        lines.append(
+            f"{info.ifname:<12} {info.operstate:<8} mac={info.mac} altnames={','.join(info.altnames) or '-'} "
+            f"addr={','.join(info.addresses) or '-'} {' '.join(gateways) or 'no default route'}"
+        )
+    return lines

ms_split_router/domains.py

@@ -0,0 +1,117 @@
+"""Derive the systemd-resolved routing-only domains (``Domains=~…``) for a split link."""
+
+import logging
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Any
+
+from .config import M365Routes, SplitLink
+from .netutil import covered_by, is_valid_domain, minimise_domains, normalise_domain, root_domain, wildcard_suffix
+from .select import matching_endpoint_sets
+
+log = logging.getLogger(__name__)
+
+# Certificate authority, OCSP/CRL and CDN domains that Microsoft lists as dependencies but does not operate. Routing
+# them to the split link would send unrelated third-party lookups to the split resolver.
+THIRD_PARTY_DOMAINS = frozenset(
+    {
+        # Certificate authorities, OCSP and CRL
+        "amazontrust.com", "comodoca.com", "digicert.com", "digicert.cn", "entrust.com", "entrust.net",
+        "geotrust.com", "globalsign.com", "globalsign.net", "godaddy.com", "identrust.com", "letsencrypt.org",
+        "lencr.org", "omniroot.com", "public-trust.com", "quovadisglobal.com", "rapidssl.com", "sectigo.com",
+        "ssl.com", "symcb.com", "symcd.com", "thawte.com", "usertrust.com", "verisign.com", "verisign.net",
+        "wosign.com",
+        # Content delivery networks
+        "akadns.net", "akamai.net", "akamaiedge.net", "akamaihd.net", "akamaized.net", "cloudflare.com",
+        "cloudflare.net", "cloudfront.net", "edgecastcdn.net", "edgekey.net", "edgesuite.net", "fastly.net",
+        "footprint.net", "llnwd.net",
+    }
+)  # fmt: skip
+
+
+@dataclass(frozen=True, slots=True)
+class DomainSelectionResult:
+    domains: list[str]
+    dropped_third_party: list[str]
+
+
+def _candidate(value: str) -> str | None:
+    """Turn an endpoint URL, wildcard or zone template into a routable suffix."""
+    domain = normalise_domain(value)
+    domain = wildcard_suffix(domain) or domain
+    return domain if is_valid_domain(domain) else None
+
+
+def _strip_privatelink(zone: str) -> str:
+    return zone.removeprefix("privatelink.")
+
+
+def _m365_domains(document: dict[str, Any], link: SplitLink) -> Iterator[str]:
+    selection = link.routes.m365 or M365Routes()
+    for endpoint_set in matching_endpoint_sets(document, selection, link.id):
+        yield from endpoint_set.get("urls", [])
+
+
+def _m365_additional_domains(document: dict[str, Any], link: SplitLink) -> Iterator[str]:
+    clouds = link.routes.m365_additional.clouds if link.routes.m365_additional else ("public",)
+    for row in document.get("m365_additional", {}).get("rows", []):
+        for cloud in clouds:
+            yield from row.get("by_cloud", {}).get(cloud, {}).get("domains", [])
+
+
+def _private_dns_domains(document: dict[str, Any], link: SplitLink) -> Iterator[str]:
+    zones = document.get("private_dns_zones", {})
+    for cloud in link.domains.private_dns_clouds:
+        entries = next((value for key, value in zones.items() if key.lower() == cloud.lower()), None)
+        if entries is None:
+            log.warning("%s: private DNS zones for %s are not in the data", link.id, cloud)
+            continue
+        for entry in entries:
+            for key in ("zones", "zone_templates"):
+                yield from (_strip_privatelink(zone) for zone in entry.get(key, []))
+            yield from entry.get("forwarders", [])
+            yield from entry.get("forwarder_templates", [])
+
+
+def derive_domains(document: dict[str, Any], link: SplitLink) -> DomainSelectionResult:
+    selection = link.domains
+    raw: list[str] = []
+    if "m365" in selection.derive_from:
+        raw.extend(_m365_domains(document, link))
+    if "m365_additional" in selection.derive_from:
+        raw.extend(_m365_additional_domains(document, link))
+    if "private_dns" in selection.derive_from:
+        raw.extend(_private_dns_domains(document, link))
+
+    derived: set[str] = set()
+    for value in raw:
+        domain = _candidate(value)
+        if domain is None:
+            log.debug("%s: skipping unusable domain %r", link.id, value)
+            continue
+        if selection.collapse_to_root:
+            domain = root_domain(domain)
+        if "." not in domain:
+            log.debug("%s: skipping single-label domain %r (from %r)", link.id, domain, value)
+            continue
+        derived.add(domain)
+
+    dropped: set[str] = set()
+    if selection.exclude_third_party:
+        dropped = {domain for domain in derived if covered_by(domain, THIRD_PARTY_DOMAINS)}
+        derived -= dropped
+
+    derived.update(selection.extra)
+    for exclude in selection.exclude:
+        derived = {domain for domain in derived if not covered_by(domain, [exclude])}
+        parents = [domain for domain in derived if covered_by(exclude, [domain])]
+        if parents:
+            log.warning(
+                "%s: excluded domain %s is still inside routed domain %s; give a default link the more specific "
+                "domain ~%s to override it",
+                link.id,
+                exclude,
+                ", ".join(sorted(parents)),
+                exclude,
+            )
+    return DomainSelectionResult(minimise_domains(derived), sorted(dropped))

ms_split_router/example_config.yaml

@@ -0,0 +1,81 @@
+# ms-split-router configuration example.
+#
+# Every key is optional unless marked "required". Values shown are the defaults unless noted.
+# Reference: https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter/blob/main/docs/configuration.md
+
+data:
+  # ms-cloud-endpoints JSON/YAML. Relative paths are resolved against this file's directory. When omitted, the
+  # generator runs `ms-cloud-endpoints` and narrows --cloud/--source/--tag/--no-ipv6 to what the links need.
+  # file: ms-cloud-endpoints.json          # uncomment to use an existing document instead of fetching
+  fetch:
+    command: ms-cloud-endpoints
+    args: []                              # extra collector arguments, e.g. [--timeout, "120"]
+
+# /etc/systemd/networkd.conf.d/10-ms-split-router.conf
+networkd_conf:
+  enabled: true
+  manage_foreign_routes: false            # keep the routes WSL installs on mirrored links
+  manage_foreign_routing_policy_rules: false
+  extra: {}                               # more keys: {Key: value} for [Network], or {Section: {Key: value}}
+
+# /etc/systemd/resolved.conf.d/10-ms-split-router.conf
+resolved_conf:
+  enabled: true
+  dnssec: false                           # DNSSEC=no; split resolvers often serve unsigned private zones
+  extra: {}                               # e.g. {LLMNR: false, MulticastDNS: false, FallbackDNS: ""}
+
+split_links:                              # required: at least one
+  - id: corp-vpn                          # required: lower-case letters, digits, '-'; used in file names
+    match:                                # exactly one of mac / altname / name
+      mac: "00:15:5d:aa:bb:cc"            # placeholder: use `ms-split-router --show-links`
+    altname_link:                         # optional: also write a .link file that gives the MAC a stable altname
+      name: msvpn
+    style: full                           # full: 10-ms-split-<id>.network | dropin: <base>.d/50-ms-split-router-<id>.conf
+    # base: 10-eth1.network               # required with style: dropin
+    gateway4: auto                        # auto (from the link's default route) | none | an IPv4 address
+    gateway6: auto                        # auto | none | an IPv6 address
+    onlink: auto                          # GatewayOnLink=; auto copies the detected route's onlink flag
+    metric: null                          # Metric= for every route
+    dns: [10.10.0.4, 10.10.0.5]           # placeholder; without DNS servers no Domains= are written
+    dns_default_route: false
+    required_for_online: "no"
+    resolve:                              # per-link resolved settings
+      dnssec: false
+      # llmnr: false
+      # multicast_dns: false
+      # dns_over_tls: false
+    domains:
+      derive_from: [m365, private_dns]    # also: m365_additional
+      collapse_to_root: true              # ~office.com rather than ~outlook.office.com
+      exclude_third_party: true           # drop CA/CDN domains (digicert.com, akamaihd.net, ...)
+      private_dns_clouds: [Commercial]
+      extra: []                           # e.g. [contoso.com] for your own zones
+      exclude: []
+    routes:
+      azure:                              # false disables Azure service tags
+        clouds: [Public]
+        service_tags: [AzureCloud]        # case-insensitive globs, e.g. [AzureActiveDirectory, "Storage.*"]
+      m365:                               # false disables Microsoft 365 endpoint sets
+        instances: [Worldwide]
+        service_areas: []                 # empty = all: Common, Exchange, SharePoint, Skype
+        categories: []                    # empty = all: Optimize, Allow, Default
+        required_only: false
+        express_route_only: false
+      m365_additional: false              # true or {clouds: [public]}: prefixes from the "other endpoints" page
+      extra_prefixes: []                  # e.g. your vNet ranges for Private Endpoints
+      exclude_prefixes: []
+      ipv6: auto                          # auto: only with an IPv6 gateway | true: required | false
+                                          # `ms-split-router --ipv6 / --no-ipv6` override this on every split link
+      widen_to:                           # widen longer prefixes to this length before collapsing (fewer routes)
+        ipv4: null                        # e.g. 24
+        ipv6: 48                          # /48 = smallest globally routed IPv6 block; null keeps prefixes exact
+
+default_links:                            # every other link that should resolve non-Microsoft names
+  - id: internet
+    match:
+      name: eth0
+    style: full                           # 20-ms-default-<id>.network
+    dns: [1.1.1.1, 1.0.0.1]
+    domains: ["~."]
+    dns_default_route: true
+    required_for_online: routable