dpyproxy 2.2.0__py3-none-any.whl

network/DomainResolver.py

@@ -0,0 +1,472 @@
+import logging
+import socket
+import time
+from dataclasses import asdict
+
+import dns
+import httpx
+from dns.exception import Timeout
+from dns.message import Message
+from dns.query import (
+    BadResponse,
+    _check_status,
+    _compute_times,
+    _destination_and_source,
+    _HTTPTransport,
+    _remaining,
+    quic,
+    receive_udp,
+    send_udp,
+    tcp,
+    tls,
+    udp,
+)
+
+from enumerators.DnsProxyMode import DnsProxyMode
+from exception.DnsException import DnsException
+from network.NetworkAddress import NetworkAddress
+from network.tcp.WrappedTcpSocket import WrappedTcpSocket
+from util.Util import parse_all_ips
+
+
+def fix_transaction_id(f):
+    """
+    Fixes the transaction id for upgraded DNS requests. DoQ specifies and DoH recommends setting the transaction id to 0
+    https://www.rfc-editor.org/rfc/rfc9250.html#section-4.2.1
+    """
+
+    def _inner(message, *args, **kwargs):
+        _id = message.id
+        # dnspython replaces message.id with 0 in this call
+        answer = f(message, *args, **kwargs)
+        answer.id = _id
+        return answer
+
+    return _inner
+
+
+class DomainResolver:
+    """
+    Resolves domains to ip addresses. Can use DNS over TLS, DNS over DoQ, DNS over UDP, DNS over TCP, DNS over TCP with
+    TCP fragmentation, and a China-specific mode that circumvents.
+    Offers static methods and non-static methods for specifiable and non-specifiable DNS resolvers and timeouts
+    respectively.
+    """
+
+    DNS_TCP_FRAG_SIZE = 20
+    THRESHOLD_CONFIRM_WORKING = 3
+    TRIES_CONFIRM_WORKING = 5
+
+    def __init__(
+        self,
+        dns_mode: DnsProxyMode,
+        resolver: NetworkAddress,
+        timeout: int,
+        hostname: str,
+        tcp_frag_size: int = DNS_TCP_FRAG_SIZE,
+        add_sni: bool = True,
+        path: str = "/dns-query",
+    ):
+        self.dns_mode = dns_mode
+        self.resolver = resolver
+        self.timeout = timeout
+        self.hostname = hostname
+        self.tcp_frag_size = tcp_frag_size
+        self.add_sni = add_sni
+        self.path = path
+
+    def to_dict(self):
+        return {
+            "dns_mode": self.dns_mode.value,
+            "resolver": asdict(self.resolver),
+            "timeout": self.timeout,
+            "hostname": self.hostname,
+            "tcp_frag_size": self.tcp_frag_size,
+            "add_sni": self.add_sni,
+            "path": self.path,
+        }
+
+    @staticmethod
+    def from_dict(data):
+        return DomainResolver(
+            dns_mode=DnsProxyMode(data["dns_mode"]),
+            resolver=NetworkAddress(**data["resolver"]),
+            timeout=data["timeout"],
+            hostname=data["hostname"],
+            tcp_frag_size=data["tcp_frag_size"],
+            add_sni=data["add_sni"],
+            path=data["path"],
+        )
+
+    @staticmethod
+    def resolve_local(domain: str) -> str:
+        """
+        Resolves the given domain to an ip address using the system's DNS resolver.
+        """
+        return socket.gethostbyname(domain)
+
+    @staticmethod
+    @fix_transaction_id
+    def resolve_dot_static(
+        message: Message, resolver: NetworkAddress, timeout: int, hostname: str, add_sni: bool = True
+    ) -> Message:
+        """
+        Resolves the given domain to an ip address using DNS over TLS on the given DNS resolver.
+        :param message: the DNS message to resolve
+        :param resolver: the DNS resolver to use
+        :param timeout: the DNS request timeout
+        :param hostname: the hostname of the DoT server, used in SNI
+        :param add_sni: whether to add SNI
+        :return: The Dns response by the resolver
+        """
+        if add_sni:
+            return tls(
+                message, where=resolver.host, port=resolver.port, timeout=timeout, server_hostname=hostname, verify=True
+            )
+        else:
+            return tls(message, where=resolver.host, port=resolver.port, timeout=timeout, verify=False)
+
+    @staticmethod
+    @fix_transaction_id
+    def resolve_doh_static(
+        message: Message,
+        resolver: NetworkAddress,
+        timeout: int,
+        hostname: str,
+        add_sni: bool = True,
+        path: str = "/dns-query",
+    ) -> Message:
+        """
+        Resolves the given domain to an ip address using DNS over HTTPS on the given DNS resolver.
+        :param message: the DNS message to resolve
+        :param resolver: the DNS resolver to use
+        :param timeout: the DNS request timeout
+        :param hostname: the hostname that TLS should use in SNI
+        :param add_sni: whether to add SNI
+        :param path: the path used in the HTTP URL
+        :return: The Dns response by the resolver
+        """
+        # Reused code from dnypython's https function below
+
+        url = f"https://{resolver.host}:{resolver.port}{path}"
+
+        (_, _, the_source) = _destination_and_source(resolver.host, resolver.port, None, 0, False)
+
+        extensions = {}
+        bootstrap_address = resolver.host
+        verify = add_sni
+        if add_sni:
+            extensions["sni_hostname"] = hostname
+        q = message
+
+        wire = q.to_wire()
+        headers = {"accept": "application/dns-message"}
+
+        if the_source is None:
+            local_address = None
+            local_port = 0
+        else:
+            local_address = the_source[0]
+            local_port = the_source[1]
+
+        transport = _HTTPTransport(
+            local_address=local_address,
+            http1=False,
+            http2=True,
+            verify=verify,
+            local_port=local_port,
+            bootstrap_address=bootstrap_address,
+            resolver=resolver,
+            family=socket.AF_UNSPEC,
+        )
+
+        cm = httpx.Client(http1=False, http2=True, verify=verify, transport=transport)
+
+        with cm as session:
+            headers.update(
+                {
+                    "content-type": "application/dns-message",
+                    "content-length": str(len(wire)),
+                }
+            )
+            response = session.post(
+                url,
+                headers=headers,
+                content=wire,
+                timeout=timeout,
+                extensions=extensions,
+            )
+
+        # status code exception
+        if response.status_code < 200 or response.status_code > 299:
+            raise ValueError(
+                f"{resolver.host} responded with status code {response.status_code}"
+                f"\nResponse headers: {response.headers}"
+                f"\nResponse body: {response.content}"
+            )
+
+        r = dns.message.from_wire(
+            response.content,
+            keyring=q.keyring,
+            request_mac=q.request_mac,
+            one_rr_per_rrset=False,
+            ignore_trailing=False,
+        )
+        r.time = response.elapsed.total_seconds()
+        if not q.is_response(r):
+            raise BadResponse
+        return r
+
+    @staticmethod
+    @fix_transaction_id
+    def resolve_doh3_static(
+        message: Message,
+        resolver: NetworkAddress,
+        timeout: int,
+        hostname: str,
+        add_sni: bool = True,
+        path: str = "/dns-query",
+    ) -> Message:
+        """
+        Resolves the given domain to an ip address using DNS over HTTP3 on the given DNS resolver.
+        :param message: the DNS message to resolve
+        :param resolver: the DNS resolver to use
+        :param timeout: the DNS request timeout
+        :param hostname: the hostname of the DNS resolver to use in SNI
+        :param add_sni: whether to add SNI
+        :param path: the path used in the HTTP URL
+        :return: The Dns response by the resolver
+        """
+        # Reused code from dnypython's _http3 function below
+
+        q = message
+        where = resolver.host
+        url = f"https://{resolver.host}:{resolver.port}{path}"
+
+        q.id = 0
+        wire = q.to_wire()
+
+        if add_sni:
+            manager = dns.quic.SyncQuicManager(verify_mode=True, server_name=hostname, h3=True)
+        else:
+            manager = dns.quic.SyncQuicManager(verify_mode=False, server_name=None, h3=True)
+
+        with manager:
+            connection = manager.connect(where, resolver.port, None, 0)
+            (start, expiration) = _compute_times(timeout)
+            with connection.make_stream(timeout) as stream:
+                stream.send_h3(url, wire, True)
+                wire = stream.receive(_remaining(expiration))
+                _check_status(stream.headers(), where, wire)
+            finish = time.time()
+
+        r = dns.message.from_wire(
+            wire,
+            keyring=q.keyring,
+            request_mac=q.request_mac,
+            one_rr_per_rrset=False,
+            ignore_trailing=False,
+        )
+        r.time = max(finish - start, 0.0)
+        if not q.is_response(r):
+            raise BadResponse
+        return r
+
+    @staticmethod
+    @fix_transaction_id
+    def resolve_doq_static(
+        message: Message, resolver: NetworkAddress, timeout: int, hostname: str, add_sni: bool = True
+    ) -> Message:
+        """
+        Resolves the given domain to an ip address using DNS over QUIC on the given DNS resolver.
+        :param message: the DNS message to resolve
+        :param resolver: the DNS resolver to use
+        :param timeout: the DNS request timeout
+        :param hostname: hostname of the DoQ server, used in SNI
+        :param add_sni: whether to add SNI
+        :return: The Dns response by the resolver
+        """
+        if add_sni:
+            return quic(
+                message,
+                where=resolver.host,
+                port=resolver.port,
+                timeout=timeout,
+                server_hostname=hostname,
+                hostname=hostname,
+                verify=True,
+            )
+        else:
+            return quic(message, where=resolver.host, port=resolver.port, timeout=timeout, verify=False)
+
+    @staticmethod
+    def resolve_udp_static_to_ip(domain: str, resolver: NetworkAddress, timeout: int) -> str:
+        """
+        Resolves the given domain to an ip address using DNS over UDP on the given DNS resolver.
+        :param domain: the domain to resolve
+        :param resolver: the DNS resolver to use
+        :param timeout: the DNS request timeout
+        :return: The resolved ip address
+        """
+        message = dns.message.make_query(domain, "A")
+        response = DomainResolver.resolve_udp_static(message, resolver, timeout)
+        ips = parse_all_ips(response)
+        if not ips or len(ips) == 0:
+            raise DnsException(f"No IPs found for domain {domain} using resolver {resolver}")
+        return ips[0]
+
+    @staticmethod
+    def resolve_udp_static(message: Message, resolver: NetworkAddress, timeout: int) -> Message:
+        """
+        Resolves the given domain to an ip address using DNS over UDP on the given DNS resolver.
+        :param message: the DNS message to resolve
+        :param resolver: the DNS resolver
+        :param timeout: the DNS request timeout
+        :return: The Dns response by the resolver
+        """
+
+        return udp(message, where=resolver.host, port=resolver.port, timeout=timeout)
+
+    @staticmethod
+    def resolve_tcp_static(message: Message, resolver: NetworkAddress, timeout: int) -> Message:
+        """
+        Resolves the given domain to an ip address using DNS over TCP on the given DNS resolver.
+        :param message: the DNS message to resolve
+        :param resolver: the DNS resolver
+        :param timeout: the DNS timeout
+        :return: The Dns response by the resolver
+        """
+        # call fragmentation method without using fragmentation
+        return DomainResolver.resolve_tcp_frag_static(message, resolver, timeout, 0)
+
+    @staticmethod
+    def resolve_tcp_frag_static(message: Message, resolver: NetworkAddress, timeout: int, frag_size: int) -> Message:
+        """
+        Resolves the given domain to an ip address using DNS over TCP with fragmentation on the given DNS resolver.
+        :param message: the DNS message to resolve
+        :param resolver: the DNS resolver to use
+        :param timeout: the DNS timeout
+        :param frag_size: size of the TCP segments used to fragment the DNS message. If 0 or negative, the message will
+        not be fragmented
+        :return: The Dns response by the resolver
+        """
+        # create fragmenting tcp socket and pass to dnspython
+        try:
+            _socket = socket.create_connection((resolver.host, resolver.port), timeout=timeout)
+        except Exception as e:
+            raise DnsException(e)
+
+        frag_socket = WrappedTcpSocket(timeout=timeout, _socket=_socket, tcp_frag_size=frag_size)
+        return tcp(message, where=resolver.host, sock=frag_socket, timeout=timeout)
+
+    @staticmethod
+    def resolve_last_response_static(message: Message, resolver: NetworkAddress, timeout: int) -> Message:
+        """
+        Resolves the given domain to an ip address over UDP but waiting a certain timeout and forwarding the last answer
+        received. This circumvents China-specific censorship.
+        :param message: the DNS message to resolve
+        :param resolver: the DNS resolver to use
+        :param timeout: timeout in seconds, the last message received in this timeout is returned
+        :return: the last DNS response message received from the server in the given timeout
+        """
+        # create udp socket
+        _socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        _socket.setblocking(False)
+        _address = (resolver.host, resolver.port)
+        # send message to server
+        send_udp(sock=_socket, what=message, destination=_address, expiration=time.time() + timeout)
+        last_received = None
+        stop_time = time.time() + timeout
+        while True:
+            try:
+                last_received, _ = receive_udp(sock=_socket, destination=_address, expiration=stop_time)
+            except Timeout:
+                break
+        return last_received
+
+    @staticmethod
+    def resolve_static(
+        mode: DnsProxyMode,
+        message: Message,
+        resolver: NetworkAddress,
+        timeout: int,
+        frag_size: int = DNS_TCP_FRAG_SIZE,
+        hostname: str = "",
+        add_sni: bool = True,
+        path: str = "/dns-query",
+    ) -> Message:
+        """
+        Resolves the requested message based on the selected mode.
+        :param mode: the DNS proxy mode to use. Must not be AUTO, will raise a DnsException
+        :param message: the DNS message to resolve
+        :param resolver: the DNS resolver
+        :param timeout: the DNS request timeout
+        :param frag_size: size of the TCP segments used to fragment the DNS message if the mode is TCP_FRAG
+        :param hostname: the hostname of the DNS resolver
+        :param add_sni: whether to add SNI
+        :param path: the path used in the HTTP URL
+        """
+        if mode == DnsProxyMode.AUTO:
+            raise DnsException("No resolution function for mode AUTO")
+        elif mode == DnsProxyMode.DOT:
+            return DomainResolver.resolve_dot_static(
+                message, resolver=resolver, timeout=timeout, hostname=hostname, add_sni=add_sni
+            )
+        elif mode == DnsProxyMode.DOH:
+            return DomainResolver.resolve_doh_static(
+                message, resolver=resolver, timeout=timeout, hostname=hostname, add_sni=add_sni, path=path
+            )
+        elif mode == DnsProxyMode.DOH3:
+            return DomainResolver.resolve_doh3_static(
+                message, resolver=resolver, timeout=timeout, hostname=hostname, add_sni=add_sni, path=path
+            )
+        elif mode == DnsProxyMode.DOQ:
+            return DomainResolver.resolve_doq_static(
+                message, resolver=resolver, timeout=timeout, hostname=hostname, add_sni=add_sni
+            )
+        elif mode == DnsProxyMode.UDP:
+            return DomainResolver.resolve_udp_static(message, resolver=resolver, timeout=timeout)
+        elif mode == DnsProxyMode.TCP:
+            return DomainResolver.resolve_tcp_static(message, resolver=resolver, timeout=timeout)
+        elif mode == DnsProxyMode.TCP_FRAG:
+            return DomainResolver.resolve_tcp_frag_static(
+                message, resolver=resolver, timeout=timeout, frag_size=frag_size
+            )
+        elif mode == DnsProxyMode.LAST_RESPONSE:
+            return DomainResolver.resolve_last_response_static(message, resolver=resolver, timeout=timeout)
+        else:
+            raise DnsException(f"Unknown mode {mode}")
+
+    def resolve(self, message: Message) -> Message:
+        """
+        Resolves the given DNS message using the configured mode on the configured provider using the configured
+        timeout and frag size.
+        """
+        return DomainResolver.resolve_static(
+            mode=self.dns_mode,
+            message=message,
+            resolver=self.resolver,
+            timeout=self.timeout,
+            frag_size=self.tcp_frag_size,
+            hostname=self.hostname,
+            add_sni=self.add_sni,
+            path=self.path,
+        )
+
+    def works(self, message: Message) -> bool:
+        """
+        Determines if the configures resolver is consistently reachable, returns true if at least 3 out of 5 connection
+        attempts worked.
+        """
+        working = 0
+        for _ in range(DomainResolver.TRIES_CONFIRM_WORKING):
+            try:
+                self.resolve(message=message)
+            except Exception as e:
+                logging.debug(f"Could not resolve to {self.resolver} for mode {self.dns_mode} with exception {e}")
+            else:
+                working += 1
+        return working >= DomainResolver.THRESHOLD_CONFIRM_WORKING
+
+    def __str__(self):
+        return f"{self.dns_mode} - {self.resolver}"

network/NetworkAddress.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class NetworkAddress:
+    host: str
+    port: int
+
+    def __str__(self):
+        return f"{self.host}:{self.port}"

network/WrappedSocket.py

@@ -0,0 +1,97 @@
+import socket
+from time import time
+
+from exception.ParserException import ParserException
+from util.constants import STANDARD_SOCKET_RECEIVE_SIZE
+
+
+class WrappedSocket:
+    """
+    Wraps a socket with useful utility functions.
+    """
+
+    def __init__(self, timeout: int, _socket: socket.socket):
+        self.timeout = timeout
+        self.buffer = b""
+        self.socket = _socket
+        self.socket.settimeout(timeout)
+
+    def read(self, size: int) -> bytes:
+        """
+        Reads specified amount of data from socket. Blocks until amount of data received or timeout.
+        :param size: Data to read.
+        :return: Read data
+        """
+        while len(self.buffer) < size:
+            self.buffer += self.socket.recv(STANDARD_SOCKET_RECEIVE_SIZE)
+        _res = self.buffer[:size]
+        self.buffer = self.buffer[size:]
+        return _res
+
+    def read_until(self, until: list[bytes], max_len: int = 100, peek: bool = False) -> bytes:
+        """
+        Returns all bytes from the socket until and including the given bytes. Also cancels after timeout
+        :param until: Bytes until which to receive
+        :param max_len: Max length until which to search
+        :param peek: Whether to keep the bytes in the buffer
+        :return: Read data
+        """
+        start_time = time()
+        while len(list(filter(lambda x: x in self.buffer, until))) == 0:
+            if len(self.buffer) > max_len:
+                raise ParserException(f"Exceeded max length of {max_len} bytes")
+            if time() - start_time > self.timeout:
+                raise ParserException(f"Exceeded timeout of {self.timeout}s")
+            self.buffer += self.socket.recv(STANDARD_SOCKET_RECEIVE_SIZE)
+        until = list(filter(lambda x: x in self.buffer, until))[0]
+        index = self.buffer.index(until) + len(until)
+        _res = self.buffer[:index]
+        if not peek:
+            self.buffer = self.buffer[index:]
+        return _res
+
+    def peek(self, size: int) -> bytes:
+        """
+        Similar to read, but keeps data in buffer.
+        """
+        while len(self.buffer) < size:
+            self.buffer += self.socket.recv(STANDARD_SOCKET_RECEIVE_SIZE)
+        return self.buffer[:size]
+
+    def recv(self, size: int, *args, **kwargs) -> bytes:
+        """
+        Works similar to recv of the wrapped socket. Prepends any bytes still buffered.
+        :param size: Size of the buffer to read into.
+        :return: Bytes read from the socket
+        """
+        if len(self.buffer) > 0:
+            _res = self.buffer[:size]
+            self.buffer = self.buffer[size:]
+        else:
+            _res = self.socket.recv(size, *args, **kwargs)
+        return _res
+
+    def close(self):
+        """
+        Closes the underlying socket.
+        """
+        self.socket.shutdown(socket.SHUT_RDWR)
+        self.socket.close()
+
+    def try_close(self):
+        """
+        Tries to close the underlying socket. If that fails, we ignore the error.
+        """
+        try:
+            self.socket.shutdown(socket.SHUT_RDWR)
+            self.socket.close()
+        except Exception:
+            pass
+
+    def inject(self, content: bytes):
+        """
+        Injects bytes to the front of the buffer. Can be used to write back read data.
+        :param content: the bytes to prepend
+        :return: None
+        """
+        self.buffer = content + self.buffer

network/protocols/Dns.py

@@ -0,0 +1,62 @@
+import dns
+from dns.message import Message, make_response
+
+from exception.DnsException import DnsException
+from network.DomainResolver import DomainResolver
+
+
+class Dns:
+    """
+    Implements methods to parse DNS messages.
+    """
+
+    DNS_MAX_SIZE = 512
+
+    @staticmethod
+    def read_dns(message: bytes) -> dns.message.Message:
+        try:
+            return dns.message.from_wire(message)
+        except Exception as e:
+            raise DnsException(f"Could not parse DNS message: {e}")
+
+    @staticmethod
+    def query_from_domain(domain: str) -> dns.message.Message:
+        domain = dns.name.from_text(domain)
+        if not domain.is_absolute():
+            domain = domain.concatenate(dns.name.root)
+
+        query = dns.message.make_query(domain, dns.rdatatype.A)
+        query.flags |= dns.flags.AD
+        query.find_rrset(query.additional, dns.name.root, 65535, dns.rdatatype.OPT, create=True, force_unique=True)
+        return query
+
+    @staticmethod
+    def ip_from_response(response: dns.message.Message, domain_resolver: DomainResolver) -> str:
+        if response.rcode() != dns.rcode.NOERROR:
+            return None
+
+        # filter ipv4 answer
+        ips = []
+        for record in response.answer:
+            if record.rdtype == dns.rdatatype.A:
+                for item in record.items:
+                    ips.append(str(item.address))
+        if len(ips) > 0:
+            return ips[0]
+        else:
+            # read CNAME hostnames from answer
+            for record in response.answer:
+                if record.rdtype == dns.rdatatype.CNAME:
+                    for item in record.items:
+                        return domain_resolver.resolve(str(item.target))
+            return None
+
+    @staticmethod
+    def make_response(message: Message, orig_id: int) -> Message:
+        """
+        Creates a DNS response message to the given query. Replaces the answers id with the original id. Necessary for
+        DoQ and maybe DoH as they set the id to 0.
+        """
+        answer = make_response(message)
+        answer.id = orig_id
+        return answer