matrix-synapse 1.143.0rc2__cp310-abi3-manylinux_2_28_aarch64.whl

synapse/http/federation/matrix_federation_agent.py

@@ -0,0 +1,490 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+import logging
+import urllib.parse
+from typing import Any, Generator
+from urllib.request import (  # type: ignore[attr-defined]
+    proxy_bypass_environment,
+)
+
+from netaddr import AddrFormatError, IPAddress, IPSet
+from zope.interface import implementer
+
+from twisted.internet import defer
+from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
+from twisted.internet.interfaces import (
+    IProtocol,
+    IProtocolFactory,
+    IReactorCore,
+    IStreamClientEndpoint,
+)
+from twisted.web.client import URI, Agent, HTTPConnectionPool
+from twisted.web.http_headers import Headers
+from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer, IResponse
+
+from synapse.config.server import ProxyConfig
+from synapse.crypto.context_factory import FederationPolicyForHTTPS
+from synapse.http import proxyagent
+from synapse.http.client import BlocklistingAgentWrapper, BlocklistingReactorWrapper
+from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint
+from synapse.http.federation.srv_resolver import Server, SrvResolver
+from synapse.http.federation.well_known_resolver import WellKnownResolver
+from synapse.http.proxyagent import ProxyAgent
+from synapse.logging.context import make_deferred_yieldable, run_in_background
+from synapse.types import ISynapseReactor
+from synapse.util.clock import Clock
+
+logger = logging.getLogger(__name__)
+
+
+@implementer(IAgent)
+class MatrixFederationAgent:
+    """An Agent-like thing which provides a `request` method which correctly
+    handles resolving matrix server names when using `matrix-federation://`. Handles
+    standard https URIs as normal. The `matrix-federation://` scheme is internal to
+    Synapse and we purposely want to avoid colliding with the `matrix://` URL scheme
+    which is now specced.
+
+    Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.)
+
+    Args:
+        reactor: twisted reactor to use for underlying requests
+
+        clock: Internal `HomeServer` clock used to track delayed and looping calls.
+            Should be obtained from `hs.get_clock()`.
+
+        tls_client_options_factory:
+            factory to use for fetching client tls options, or none to disable TLS.
+
+        user_agent:
+            The user agent header to use for federation requests.
+
+        ip_allowlist: Allowed IP addresses.
+
+        ip_blocklist: Disallowed IP addresses.
+
+        proxy_config: Proxy configuration to use for this agent.
+
+        proxy_reactor: twisted reactor to use for connections to the proxy server
+           reactor might have some blocking applied (i.e. for DNS queries),
+           but we need unblocked access to the proxy.
+
+        _srv_resolver:
+            SrvResolver implementation to use for looking up SRV records. None
+            to use a default implementation.
+
+        _well_known_resolver:
+            WellKnownResolver to use to perform well-known lookups. None to use a
+            default implementation.
+    """
+
+    def __init__(
+        self,
+        *,
+        server_name: str,
+        reactor: ISynapseReactor,
+        clock: Clock,
+        tls_client_options_factory: FederationPolicyForHTTPS | None,
+        user_agent: bytes,
+        ip_allowlist: IPSet | None,
+        ip_blocklist: IPSet,
+        proxy_config: ProxyConfig | None = None,
+        _srv_resolver: SrvResolver | None = None,
+        _well_known_resolver: WellKnownResolver | None = None,
+    ):
+        """
+        Args:
+            server_name: Our homeserver name (used to label metrics) (`hs.hostname`).
+            reactor
+            clock: Should be the `hs` clock from `hs.get_clock()`
+            tls_client_options_factory
+            user_agent
+            ip_allowlist
+            ip_blocklist
+            _srv_resolver
+            _well_known_resolver
+        """
+
+        # proxy_reactor is not blocklisting reactor
+        proxy_reactor = reactor
+
+        # We need to use a DNS resolver which filters out blocked IP
+        # addresses, to prevent DNS rebinding.
+        reactor = BlocklistingReactorWrapper(reactor, ip_allowlist, ip_blocklist)
+
+        self._pool = HTTPConnectionPool(reactor)
+        self._pool.retryAutomatically = False
+        self._pool.maxPersistentPerHost = 5
+        self._pool.cachedConnectionTimeout = 2 * 60
+
+        self._agent = Agent.usingEndpointFactory(
+            reactor,
+            MatrixHostnameEndpointFactory(
+                reactor=reactor,
+                proxy_reactor=proxy_reactor,
+                tls_client_options_factory=tls_client_options_factory,
+                srv_resolver=_srv_resolver,
+                proxy_config=proxy_config,
+            ),
+            pool=self._pool,
+        )
+        self.user_agent = user_agent
+
+        if _well_known_resolver is None:
+            _well_known_resolver = WellKnownResolver(
+                server_name=server_name,
+                reactor=reactor,
+                clock=clock,
+                agent=BlocklistingAgentWrapper(
+                    ProxyAgent(
+                        reactor=reactor,
+                        proxy_reactor=proxy_reactor,
+                        pool=self._pool,
+                        contextFactory=tls_client_options_factory,
+                        proxy_config=proxy_config,
+                    ),
+                    ip_blocklist=ip_blocklist,
+                ),
+                user_agent=self.user_agent,
+            )
+
+        self._well_known_resolver = _well_known_resolver
+
+    @defer.inlineCallbacks
+    def request(
+        self,
+        method: bytes,
+        uri: bytes,
+        headers: Headers | None = None,
+        bodyProducer: IBodyProducer | None = None,
+    ) -> Generator[defer.Deferred, Any, IResponse]:
+        """
+        Args:
+            method: HTTP method: GET/POST/etc
+            uri: Absolute URI to be retrieved
+            headers:
+                HTTP headers to send with the request, or None to send no extra headers.
+            bodyProducer:
+                An object which can generate bytes to make up the
+                body of this request (for example, the properly encoded contents of
+                a file for a file upload).  Or None if the request is to have
+                no body.
+        Returns:
+            A deferred which fires when the header of the response has been received
+            (regardless of the response status code). Fails if there is any problem
+            which prevents that response from being received (including problems that
+            prevent the request from being sent).
+        """
+        # We use urlparse as that will set `port` to None if there is no
+        # explicit port.
+        parsed_uri = urllib.parse.urlparse(uri)
+
+        # There must be a valid hostname.
+        assert parsed_uri.hostname
+
+        # If this is a matrix-federation:// URI check if the server has delegated matrix
+        # traffic using well-known delegation.
+        #
+        # We have to do this here and not in the endpoint as we need to rewrite
+        # the host header with the delegated server name.
+        delegated_server = None
+        if (
+            parsed_uri.scheme == b"matrix-federation"
+            and not _is_ip_literal(parsed_uri.hostname)
+            and not parsed_uri.port
+        ):
+            well_known_result = yield defer.ensureDeferred(
+                self._well_known_resolver.get_well_known(parsed_uri.hostname)
+            )
+            delegated_server = well_known_result.delegated_server
+
+        if delegated_server:
+            # Ok, the server has delegated matrix traffic to somewhere else, so
+            # lets rewrite the URL to replace the server with the delegated
+            # server name.
+            uri = urllib.parse.urlunparse(
+                (
+                    parsed_uri.scheme,
+                    delegated_server,
+                    parsed_uri.path,
+                    parsed_uri.params,
+                    parsed_uri.query,
+                    parsed_uri.fragment,
+                )
+            )
+            parsed_uri = urllib.parse.urlparse(uri)
+
+        # We need to make sure the host header is set to the netloc of the
+        # server and that a user-agent is provided.
+        if headers is None:
+            request_headers = Headers()
+        else:
+            request_headers = headers.copy()
+
+        if not request_headers.hasHeader(b"host"):
+            request_headers.addRawHeader(b"host", parsed_uri.netloc)
+        if not request_headers.hasHeader(b"user-agent"):
+            request_headers.addRawHeader(b"user-agent", self.user_agent)
+
+        res = yield make_deferred_yieldable(
+            self._agent.request(method, uri, request_headers, bodyProducer)
+        )
+
+        return res
+
+
+@implementer(IAgentEndpointFactory)
+class MatrixHostnameEndpointFactory:
+    """Factory for MatrixHostnameEndpoint for parsing to an Agent."""
+
+    def __init__(
+        self,
+        *,
+        reactor: IReactorCore,
+        proxy_reactor: IReactorCore,
+        tls_client_options_factory: FederationPolicyForHTTPS | None,
+        srv_resolver: SrvResolver | None,
+        proxy_config: ProxyConfig | None,
+    ):
+        self._reactor = reactor
+        self._proxy_reactor = proxy_reactor
+        self._tls_client_options_factory = tls_client_options_factory
+        self._proxy_config = proxy_config
+
+        if srv_resolver is None:
+            srv_resolver = SrvResolver()
+
+        self._srv_resolver = srv_resolver
+
+    def endpointForURI(self, parsed_uri: URI) -> "MatrixHostnameEndpoint":
+        return MatrixHostnameEndpoint(
+            reactor=self._reactor,
+            proxy_reactor=self._proxy_reactor,
+            tls_client_options_factory=self._tls_client_options_factory,
+            srv_resolver=self._srv_resolver,
+            proxy_config=self._proxy_config,
+            parsed_uri=parsed_uri,
+        )
+
+
+@implementer(IStreamClientEndpoint)
+class MatrixHostnameEndpoint:
+    """An endpoint that resolves matrix-federation:// URLs using Matrix server name
+    resolution (i.e. via SRV). Does not check for well-known delegation.
+
+    Args:
+        reactor: twisted reactor to use for underlying requests
+        proxy_reactor: twisted reactor to use for connections to the proxy server.
+           'reactor' might have some blocking applied (i.e. for DNS queries),
+           but we need unblocked access to the proxy.
+        tls_client_options_factory:
+            factory to use for fetching client tls options, or none to disable TLS.
+        srv_resolver: The SRV resolver to use
+        proxy_config: Proxy configuration to use for this agent.
+        parsed_uri: The parsed URI that we're wanting to connect to.
+
+    Raises:
+        ValueError if the environment variables contain an invalid proxy specification.
+        RuntimeError if no tls_options_factory is given for a https connection
+    """
+
+    def __init__(
+        self,
+        *,
+        reactor: IReactorCore,
+        proxy_reactor: IReactorCore,
+        tls_client_options_factory: FederationPolicyForHTTPS | None,
+        srv_resolver: SrvResolver,
+        proxy_config: ProxyConfig | None,
+        parsed_uri: URI,
+    ):
+        self._reactor = reactor
+        self._parsed_uri = parsed_uri
+        self.proxy_config = proxy_config
+
+        # http_proxy is not needed because federation is always over TLS
+
+        # endpoint and credentials to use to connect to the outbound https proxy, if any.
+        (
+            self._https_proxy_endpoint,
+            self._https_proxy_creds,
+        ) = proxyagent.http_proxy_endpoint(
+            self.proxy_config.https_proxy.encode()
+            if self.proxy_config and self.proxy_config.https_proxy
+            else None,
+            proxy_reactor,
+            tls_client_options_factory,
+        )
+
+        # set up the TLS connection params
+        #
+        # XXX disabling TLS is really only supported here for the benefit of the
+        # unit tests. We should make the UTs cope with TLS rather than having to make
+        # the code support the unit tests.
+
+        if tls_client_options_factory is None:
+            self._tls_options = None
+        else:
+            self._tls_options = tls_client_options_factory.get_options(
+                self._parsed_uri.host
+            )
+
+        self._srv_resolver = srv_resolver
+
+    def connect(
+        self, protocol_factory: IProtocolFactory
+    ) -> "defer.Deferred[IProtocol]":
+        """Implements IStreamClientEndpoint interface"""
+
+        return run_in_background(self._do_connect, protocol_factory)
+
+    async def _do_connect(self, protocol_factory: IProtocolFactory) -> IProtocol:
+        first_exception = None
+
+        server_list = await self._resolve_server()
+
+        for server in server_list:
+            host = server.host
+            port = server.port
+
+            should_skip_proxy = False
+            if self.proxy_config is not None:
+                should_skip_proxy = proxy_bypass_environment(
+                    host.decode(),
+                    proxies=self.proxy_config.get_proxies_dictionary(),
+                )
+
+            endpoint: IStreamClientEndpoint
+            try:
+                if self._https_proxy_endpoint and not should_skip_proxy:
+                    logger.debug(
+                        "Connecting to %s:%i via %s",
+                        host.decode("ascii"),
+                        port,
+                        self._https_proxy_endpoint,
+                    )
+                    endpoint = HTTPConnectProxyEndpoint(
+                        self._reactor,
+                        self._https_proxy_endpoint,
+                        host,
+                        port,
+                        proxy_creds=self._https_proxy_creds,
+                    )
+                else:
+                    logger.debug("Connecting to %s:%i", host.decode("ascii"), port)
+                    # not using a proxy
+                    endpoint = HostnameEndpoint(self._reactor, host, port)
+                if self._tls_options:
+                    endpoint = wrapClientTLS(self._tls_options, endpoint)
+                result = await make_deferred_yieldable(
+                    endpoint.connect(protocol_factory)
+                )
+
+                return result
+            except Exception as e:
+                logger.info(
+                    "Failed to connect to %s:%i: %s", host.decode("ascii"), port, e
+                )
+                if not first_exception:
+                    first_exception = e
+
+        # We return the first failure because that's probably the most interesting.
+        if first_exception:
+            raise first_exception
+
+        # This shouldn't happen as we should always have at least one host/port
+        # to try and if that doesn't work then we'll have an exception.
+        raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc,))
+
+    async def _resolve_server(self) -> list[Server]:
+        """Resolves the server name to a list of hosts and ports to attempt to
+        connect to.
+        """
+
+        if self._parsed_uri.scheme != b"matrix-federation":
+            return [Server(host=self._parsed_uri.host, port=self._parsed_uri.port)]
+
+        # Note: We don't do well-known lookup as that needs to have happened
+        # before now, due to needing to rewrite the Host header of the HTTP
+        # request.
+
+        # We reparse the URI so that defaultPort is -1 rather than 80
+        parsed_uri = urllib.parse.urlparse(self._parsed_uri.toBytes())
+
+        host = parsed_uri.hostname
+        port = parsed_uri.port
+
+        # If there is an explicit port or the host is an IP address we bypass
+        # SRV lookups and just use the given host/port.
+        if port or _is_ip_literal(host):
+            return [Server(host, port or 8448)]
+
+        # Check _matrix-fed._tcp SRV record.
+        logger.debug("Looking up SRV record for %s", host.decode(errors="replace"))
+        server_list = await self._srv_resolver.resolve_service(
+            b"_matrix-fed._tcp." + host
+        )
+
+        if server_list:
+            if logger.isEnabledFor(logging.DEBUG):
+                logger.debug(
+                    "Got %s from SRV lookup for %s",
+                    ", ".join(map(str, server_list)),
+                    host.decode(errors="replace"),
+                )
+            return server_list
+
+        # No _matrix-fed._tcp SRV record, fallback to legacy _matrix._tcp SRV record.
+        logger.debug(
+            "Looking up deprecated SRV record for %s", host.decode(errors="replace")
+        )
+        server_list = await self._srv_resolver.resolve_service(b"_matrix._tcp." + host)
+
+        if server_list:
+            if logger.isEnabledFor(logging.DEBUG):
+                logger.debug(
+                    "Got %s from deprecated SRV lookup for %s",
+                    ", ".join(map(str, server_list)),
+                    host.decode(errors="replace"),
+                )
+            return server_list
+
+        # No SRV records, so we fallback to host and 8448
+        logger.debug("No SRV records for %s", host.decode(errors="replace"))
+        return [Server(host, 8448)]
+
+
+def _is_ip_literal(host: bytes) -> bool:
+    """Test if the given host name is either an IPv4 or IPv6 literal.
+
+    Args:
+        host: The host name to check
+
+    Returns:
+        True if the hostname is an IP address literal.
+    """
+
+    host_str = host.decode("ascii")
+
+    try:
+        IPAddress(host_str)
+        return True
+    except AddrFormatError:
+        return False

synapse/http/federation/srv_resolver.py

@@ -0,0 +1,196 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import logging
+import random
+import time
+from typing import Any, Callable
+
+import attr
+
+from twisted.internet.error import ConnectError
+from twisted.names import client, dns
+from twisted.names.error import DNSNameError, DNSNotImplementedError, DomainError
+
+from synapse.logging.context import make_deferred_yieldable
+
+logger = logging.getLogger(__name__)
+
+SERVER_CACHE: dict[bytes, list["Server"]] = {}
+
+
+@attr.s(auto_attribs=True, slots=True, frozen=True)
+class Server:
+    """
+    Our record of an individual server which can be tried to reach a destination.
+
+    Attributes:
+        host: target hostname
+        port:
+        priority:
+        weight:
+        expires: when the cache should expire this record - in *seconds* since
+            the epoch
+    """
+
+    host: bytes
+    port: int
+    priority: int = 0
+    weight: int = 0
+    expires: int = 0
+
+
+def _sort_server_list(server_list: list[Server]) -> list[Server]:
+    """Given a list of SRV records sort them into priority order and shuffle
+    each priority with the given weight.
+    """
+    priority_map: dict[int, list[Server]] = {}
+
+    for server in server_list:
+        priority_map.setdefault(server.priority, []).append(server)
+
+    results = []
+    for priority in sorted(priority_map):
+        servers = priority_map[priority]
+
+        # This algorithms roughly follows the algorithm described in RFC2782,
+        # changed to remove an off-by-one error.
+        #
+        # N.B. Weights can be zero, which means that they should be picked
+        # rarely.
+
+        total_weight = sum(s.weight for s in servers)
+
+        # Total weight can become zero if there are only zero weight servers
+        # left, which we handle by just shuffling and appending to the results.
+        while servers and total_weight:
+            target_weight = random.randint(1, total_weight)
+
+            for s in servers:
+                target_weight -= s.weight
+
+                if target_weight <= 0:
+                    break
+
+            results.append(s)
+            servers.remove(s)
+            total_weight -= s.weight
+
+        if servers:
+            random.shuffle(servers)
+            results.extend(servers)
+
+    return results
+
+
+class SrvResolver:
+    """Interface to the dns client to do SRV lookups, with result caching.
+
+    The default resolver in twisted.names doesn't do any caching (it has a CacheResolver,
+    but the cache never gets populated), so we add our own caching layer here.
+
+    Args:
+        dns_client (twisted.internet.interfaces.IResolver): twisted resolver impl
+        cache: cache object
+        get_time: clock implementation. Should return seconds since the epoch
+    """
+
+    def __init__(
+        self,
+        dns_client: Any = client,
+        cache: dict[bytes, list[Server]] = SERVER_CACHE,
+        get_time: Callable[[], float] = time.time,
+    ):
+        self._dns_client = dns_client
+        self._cache = cache
+        self._get_time = get_time
+
+    async def resolve_service(self, service_name: bytes) -> list[Server]:
+        """Look up a SRV record
+
+        Args:
+            service_name: record to look up
+
+        Returns:
+            a list of the SRV records, or an empty list if none found
+        """
+        now = int(self._get_time())
+
+        if not isinstance(service_name, bytes):
+            raise TypeError("%r is not a byte string" % (service_name,))
+
+        cache_entry = self._cache.get(service_name, None)
+        if cache_entry:
+            if all(s.expires > now for s in cache_entry):
+                servers = list(cache_entry)
+                return _sort_server_list(servers)
+
+        try:
+            answers, _, _ = await make_deferred_yieldable(
+                self._dns_client.lookupService(service_name)
+            )
+        except DNSNameError:
+            # TODO: cache this. We can get the SOA out of the exception, and use
+            # the negative-TTL value.
+            return []
+        except DNSNotImplementedError:
+            # For .onion homeservers this is unavailable, just fallback to host:8448
+            return []
+        except DomainError as e:
+            # We failed to resolve the name (other than a NameError)
+            # Try something in the cache, else rereaise
+            cache_entry = self._cache.get(service_name, None)
+            if cache_entry:
+                logger.warning(
+                    "Failed to resolve %r, falling back to cache. %r", service_name, e
+                )
+                return list(cache_entry)
+            else:
+                raise e
+
+        if (
+            len(answers) == 1
+            and answers[0].type == dns.SRV
+            and answers[0].payload
+            and answers[0].payload.target == dns.Name(b".")
+        ):
+            raise ConnectError(f"Service {service_name!r} unavailable")
+
+        servers = []
+
+        for answer in answers:
+            if answer.type != dns.SRV or not answer.payload:
+                continue
+
+            payload = answer.payload
+
+            servers.append(
+                Server(
+                    host=payload.target.name,
+                    port=payload.port,
+                    priority=payload.priority,
+                    weight=payload.weight,
+                    expires=now + answer.ttl,
+                )
+            )
+
+        self._cache[service_name] = list(servers)
+        return _sort_server_list(servers)