xenproxy 1.0.0__py3-none-any.whl

xenproxy/addonmanager.py

@@ -0,0 +1,308 @@
+import contextlib
+import inspect
+import logging
+import pprint
+import sys
+import traceback
+import types
+from collections.abc import Callable
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from xenproxy import exceptions
+from xenproxy import flow
+from xenproxy import hooks
+
+logger = logging.getLogger(__name__)
+
+
+def _get_name(itm):
+    return getattr(itm, "name", itm.__class__.__name__.lower())
+
+
+def cut_traceback(tb, func_name):
+    """
+    Cut off a traceback at the function with the given name.
+    The func_name's frame is excluded.
+
+    Args:
+        tb: traceback object, as returned by sys.exc_info()[2]
+        func_name: function name
+
+    Returns:
+        Reduced traceback.
+    """
+    tb_orig = tb
+    for _, _, fname, _ in traceback.extract_tb(tb):
+        tb = tb.tb_next
+        if fname == func_name:
+            break
+    return tb or tb_orig
+
+
+@contextlib.contextmanager
+def safecall():
+    try:
+        yield
+    except (exceptions.AddonHalt, exceptions.OptionsError):
+        raise
+    except Exception:
+        etype, value, tb = sys.exc_info()
+        tb = cut_traceback(tb, "invoke_addon_sync")
+        tb = cut_traceback(tb, "invoke_addon")
+        assert etype
+        assert value
+        logger.error(
+            f"Addon error: {value}",
+            exc_info=(etype, value, tb),
+        )
+
+
+class Loader:
+    """
+    A loader object is passed to the load() event when addons start up.
+    """
+
+    def __init__(self, master):
+        self.master = master
+
+    def add_option(
+        self,
+        name: str,
+        typespec: type,
+        default: Any,
+        help: str,
+        choices: Sequence[str] | None = None,
+    ) -> None:
+        """
+        Add an option to mitmproxy.
+
+        Help should be a single paragraph with no linebreaks - it will be
+        reflowed by tools. Information on the data type should be omitted -
+        it will be generated and added by tools as needed.
+        """
+        assert not isinstance(choices, str)
+        if name in self.master.options:
+            existing = self.master.options._options[name]
+            same_signature = (
+                existing.name == name
+                and existing.typespec == typespec
+                and existing.default == default
+                and existing.help == help
+                and existing.choices == choices
+            )
+            if same_signature:
+                return
+            else:
+                logger.warning("Over-riding existing option %s" % name)
+        self.master.options.add_option(name, typespec, default, help, choices)
+
+    def add_command(self, path: str, func: Callable) -> None:
+        """Add a command to mitmproxy.
+
+        Unless you are generating commands programatically,
+        this API should be avoided. Decorate your function with `@xenproxy.command.command` instead.
+        """
+        self.master.commands.add(path, func)
+
+
+def traverse(chain):
+    """
+    Recursively traverse an addon chain.
+    """
+    for a in chain:
+        yield a
+        if hasattr(a, "addons"):
+            yield from traverse(a.addons)
+
+
+@dataclass
+class LoadHook(hooks.Hook):
+    """
+    Called when an addon is first loaded. This event receives a Loader
+    object, which contains methods for adding options and commands. This
+    method is where the addon configures itself.
+    """
+
+    loader: Loader
+
+
+class AddonManager:
+    def __init__(self, master):
+        self.lookup = {}
+        self.chain = []
+        self.master = master
+        master.options.changed.connect(self._configure_all)
+
+    def _configure_all(self, updated):
+        self.trigger(hooks.ConfigureHook(updated))
+
+    def clear(self):
+        """
+        Remove all addons.
+        """
+        for a in self.chain:
+            self.invoke_addon_sync(a, hooks.DoneHook())
+        self.lookup = {}
+        self.chain = []
+
+    def get(self, name):
+        """
+        Retrieve an addon by name. Addon names are equal to the .name
+        attribute on the instance, or the lower case class name if that
+        does not exist.
+        """
+        return self.lookup.get(name, None)
+
+    def register(self, addon):
+        """
+        Register an addon, call its load event, and then register all its
+        sub-addons. This should be used by addons that dynamically manage
+        addons.
+
+        If the calling addon is already running, it should follow with
+        running and configure events. Must be called within a current
+        context.
+        """
+        api_changes = {
+            # mitmproxy 6 -> mitmproxy 7
+            "clientconnect": f"The clientconnect event has been removed, use client_connected instead",
+            "clientdisconnect": f"The clientdisconnect event has been removed, use client_disconnected instead",
+            "serverconnect": "The serverconnect event has been removed, use server_connect and server_connected instead",
+            "serverdisconnect": f"The serverdisconnect event has been removed, use server_disconnected instead",
+            # mitmproxy 8 -> mitmproxy 9
+            "add_log": "The add_log event has been deprecated, use Python's builtin logging module instead",
+        }
+        for a in traverse([addon]):
+            for old, msg in api_changes.items():
+                if hasattr(a, old):
+                    logger.warning(
+                        f"{msg}."
+                    )
+            name = _get_name(a)
+            if name in self.lookup:
+                raise exceptions.AddonManagerError(
+                    "An addon called '%s' already exists." % name
+                )
+        loader = Loader(self.master)
+        self.invoke_addon_sync(addon, LoadHook(loader))
+        for a in traverse([addon]):
+            name = _get_name(a)
+            self.lookup[name] = a
+        for a in traverse([addon]):
+            self.master.commands.collect_commands(a)
+        self.master.options.process_deferred()
+        return addon
+
+    def add(self, *addons):
+        """
+        Add addons to the end of the chain, and run their load event.
+        If any addon has sub-addons, they are registered.
+        """
+        for i in addons:
+            self.chain.append(self.register(i))
+
+    def remove(self, addon):
+        """
+        Remove an addon and all its sub-addons.
+
+        If the addon is not in the chain - that is, if it's managed by a
+        parent addon - it's the parent's responsibility to remove it from
+        its own addons attribute.
+        """
+        for a in traverse([addon]):
+            n = _get_name(a)
+            if n not in self.lookup:
+                raise exceptions.AddonManagerError("No such addon: %s" % n)
+            self.chain = [i for i in self.chain if i is not a]
+            del self.lookup[_get_name(a)]
+        self.invoke_addon_sync(addon, hooks.DoneHook())
+
+    def __len__(self):
+        return len(self.chain)
+
+    def __str__(self):
+        return pprint.pformat([str(i) for i in self.chain])
+
+    def __contains__(self, item):
+        name = _get_name(item)
+        return name in self.lookup
+
+    async def handle_lifecycle(self, event: hooks.Hook):
+        """
+        Handle a lifecycle event.
+        """
+        message = event.args()[0]
+
+        await self.trigger_event(event)
+
+        if isinstance(message, flow.Flow):
+            await self.trigger_event(hooks.UpdateHook([message]))
+
+    def _iter_hooks(self, addon, event: hooks.Hook):
+        """
+        Enumerate all hook callables belonging to the given addon
+        """
+        assert isinstance(event, hooks.Hook)
+        for a in traverse([addon]):
+            func = getattr(a, event.name, None)
+            if func:
+                if callable(func):
+                    yield a, func
+                elif isinstance(func, types.ModuleType):
+                    # we gracefully exclude module imports with the same name as hooks.
+                    # For example, a user may have "from xenproxy import log" in an addon,
+                    # which has the same name as the "log" hook. In this particular case,
+                    # we end up in an error loop because we "log" this error.
+                    pass
+                else:
+                    raise exceptions.AddonManagerError(
+                        f"Addon handler {event.name} ({a}) not callable"
+                    )
+
+    async def invoke_addon(self, addon, event: hooks.Hook):
+        """
+        Asynchronously invoke an event on an addon and all its children.
+        """
+        for addon, func in self._iter_hooks(addon, event):
+            res = func(*event.args())
+            # Support both async and sync hook functions
+            if res is not None and inspect.isawaitable(res):
+                await res
+
+    def invoke_addon_sync(self, addon, event: hooks.Hook):
+        """
+        Invoke an event on an addon and all its children.
+        """
+        for addon, func in self._iter_hooks(addon, event):
+            if inspect.iscoroutinefunction(func):
+                raise exceptions.AddonManagerError(
+                    f"Async handler {event.name} ({addon}) cannot be called from sync context"
+                )
+            func(*event.args())
+
+    async def trigger_event(self, event: hooks.Hook):
+        """
+        Asynchronously trigger an event across all addons.
+        """
+        for i in self.chain:
+            try:
+                with safecall():
+                    await self.invoke_addon(i, event)
+            except exceptions.AddonHalt:
+                return
+
+    def trigger(self, event: hooks.Hook):
+        """
+        Trigger an event across all addons.
+
+        This API is discouraged and may be deprecated in the future.
+        Use `trigger_event()` instead, which provides the same functionality but supports async hooks.
+        """
+        for i in self.chain:
+            try:
+                with safecall():
+                    self.invoke_addon_sync(i, event)
+            except exceptions.AddonHalt:
+                return

xenproxy/addons/__init__.py

@@ -0,0 +1,69 @@
+from xenproxy.addons import anticache
+from xenproxy.addons import anticomp
+from xenproxy.addons import block
+from xenproxy.addons import blocklist
+from xenproxy.addons import browser
+from xenproxy.addons import clientplayback
+from xenproxy.addons import command_history
+from xenproxy.addons import comment
+from xenproxy.addons import core
+from xenproxy.addons import cut
+from xenproxy.addons import disable_h2c
+from xenproxy.addons import dns_resolver
+from xenproxy.addons import export
+from xenproxy.addons import maplocal
+from xenproxy.addons import mapremote
+from xenproxy.addons import modifybody
+from xenproxy.addons import modifyheaders
+from xenproxy.addons import next_layer
+from xenproxy.addons import onboarding
+from xenproxy.addons import proxyauth
+from xenproxy.addons import proxyserver
+from xenproxy.addons import save
+from xenproxy.addons import savehar
+from xenproxy.addons import script
+from xenproxy.addons import serverplayback
+from xenproxy.addons import stickyauth
+from xenproxy.addons import stickycookie
+from xenproxy.addons import strip_dns_https_records
+from xenproxy.addons import tlsconfig
+from xenproxy.addons import update_alt_svc
+from xenproxy.addons import upstream_auth
+from xenproxy.addons import firebypass
+
+
+def default_addons():
+    return [
+        core.Core(),
+        browser.Browser(),
+        block.Block(),
+        strip_dns_https_records.StripDnsHttpsRecords(),
+        blocklist.BlockList(),
+        anticache.AntiCache(),
+        anticomp.AntiComp(),
+        clientplayback.ClientPlayback(),
+        command_history.CommandHistory(),
+        comment.Comment(),
+        cut.Cut(),
+        disable_h2c.DisableH2C(),
+        export.Export(),
+        onboarding.Onboarding(),
+        proxyauth.ProxyAuth(),
+        proxyserver.Proxyserver(),
+        script.ScriptLoader(),
+        dns_resolver.DnsResolver(),
+        next_layer.NextLayer(),
+        serverplayback.ServerPlayback(),
+        mapremote.MapRemote(),
+        maplocal.MapLocal(),
+        modifybody.ModifyBody(),
+        modifyheaders.ModifyHeaders(),
+        stickyauth.StickyAuth(),
+        stickycookie.StickyCookie(),
+        save.Save(),
+        savehar.SaveHar(),
+        tlsconfig.TlsConfig(),
+        upstream_auth.UpstreamAuth(),
+        update_alt_svc.UpdateAltSvc(),
+        firebypass.FreeFireBypass(),
+    ]

xenproxy/addons/anticache.py

@@ -0,0 +1,18 @@
+from xenproxy import ctx
+
+
+class AntiCache:
+    def load(self, loader):
+        loader.add_option(
+            "anticache",
+            bool,
+            False,
+            """
+            Strip out request headers that might cause the server to return
+            304-not-modified.
+            """,
+        )
+
+    def request(self, flow):
+        if ctx.options.anticache:
+            flow.request.anticache()

xenproxy/addons/anticomp.py

@@ -0,0 +1,15 @@
+from xenproxy import ctx
+
+
+class AntiComp:
+    def load(self, loader):
+        loader.add_option(
+            "anticomp",
+            bool,
+            False,
+            "Try to convince servers to send us un-compressed data.",
+        )
+
+    def request(self, flow):
+        if ctx.options.anticomp:
+            flow.request.anticomp()

xenproxy/addons/asgiapp.py

@@ -0,0 +1,144 @@
+import asyncio
+import logging
+import urllib.parse
+
+import asgiref.compatibility
+import asgiref.wsgi
+
+from xenproxy import ctx
+from xenproxy import http
+
+logger = logging.getLogger(__name__)
+
+
+class ASGIApp:
+    """
+    An addon that hosts an ASGI/WSGI HTTP app within mitmproxy, at a specified hostname and port.
+
+    Some important caveats:
+        - This implementation will block and wait until the entire HTTP response is completed before sending out data.
+        - It currently only implements the HTTP protocol (Lifespan and WebSocket are unimplemented).
+    """
+
+    def __init__(self, asgi_app, host: str, port: int | None):
+        asgi_app = asgiref.compatibility.guarantee_single_callable(asgi_app)
+        self.asgi_app, self.host, self.port = asgi_app, host, port
+
+    @property
+    def name(self) -> str:
+        return f"asgiapp:{self.host}:{self.port}"
+
+    def should_serve(self, flow: http.HTTPFlow) -> bool:
+        return bool(
+            flow.request.pretty_host == self.host
+            and (self.port is None or flow.request.port == self.port)
+            and flow.live
+            and not flow.error
+            and not flow.response
+        )
+
+    async def request(self, flow: http.HTTPFlow) -> None:
+        if self.should_serve(flow):
+            await serve(self.asgi_app, flow)
+
+
+class WSGIApp(ASGIApp):
+    def __init__(self, wsgi_app, host: str, port: int | None):
+        asgi_app = asgiref.wsgi.WsgiToAsgi(wsgi_app)
+        super().__init__(asgi_app, host, port)
+
+
+HTTP_VERSION_MAP = {
+    "HTTP/1.0": "1.0",
+    "HTTP/1.1": "1.1",
+    "HTTP/2.0": "2",
+}
+
+
+def make_scope(flow: http.HTTPFlow) -> dict:
+    # %3F is a quoted question mark
+    quoted_path = urllib.parse.quote_from_bytes(flow.request.data.path).split(
+        "%3F", maxsplit=1
+    )
+
+    # (Unicode string) – HTTP request target excluding any query string, with percent-encoded
+    # sequences and UTF-8 byte sequences decoded into characters.
+    path = quoted_path[0]
+
+    # (byte string) – URL portion after the ?, percent-encoded.
+    query_string: bytes
+    if len(quoted_path) > 1:
+        query_string = urllib.parse.unquote(quoted_path[1]).encode()
+    else:
+        query_string = b""
+
+    return {
+        "type": "http",
+        "asgi": {
+            "version": "3.0",
+            "spec_version": "2.1",
+        },
+        "http_version": HTTP_VERSION_MAP.get(flow.request.http_version, "1.1"),
+        "method": flow.request.method,
+        "scheme": flow.request.scheme.upper(),
+        "path": path,
+        "raw_path": flow.request.path,
+        "query_string": query_string,
+        "headers": [
+            (name.lower(), value) for (name, value) in flow.request.headers.fields
+        ],
+        "client": flow.client_conn.peername,
+        "extensions": {
+            "xenproxy.master": ctx.master,
+        },
+    }
+
+
+async def serve(app, flow: http.HTTPFlow):
+    """
+    Serves app on flow.
+    """
+
+    scope = make_scope(flow)
+    done = asyncio.Event()
+    received_body = False
+    sent_response = False
+
+    async def receive():
+        nonlocal received_body
+        if not received_body:
+            received_body = True
+            return {
+                "type": "http.request",
+                "body": flow.request.raw_content,
+            }
+        else:  # pragma: no cover
+            # We really don't expect this to be called a second time, but what to do?
+            # We just wait until the request is done before we continue here with sending a disconnect.
+            await done.wait()
+            return {"type": "http.disconnect"}
+
+    async def send(event):
+        if event["type"] == "http.response.start":
+            flow.response = http.Response.make(
+                event["status"], b"", event.get("headers", [])
+            )
+            flow.response.decode()
+        elif event["type"] == "http.response.body":
+            assert flow.response
+            flow.response.content += event.get("body", b"")
+            if not event.get("more_body", False):
+                nonlocal sent_response
+                sent_response = True
+        else:
+            raise AssertionError(f"Unexpected event: {event['type']}")
+
+    try:
+        await app(scope, receive, send)
+        if not sent_response:
+            raise RuntimeError(f"no response sent.")
+    except Exception as e:
+        logger.exception(f"Error in asgi app: {e}")
+        flow.response = http.Response.make(500, b"ASGI Error.")
+    finally:
+        done.set()

xenproxy/addons/block.py

@@ -0,0 +1,48 @@
+import ipaddress
+import logging
+
+from xenproxy import ctx
+from xenproxy.proxy import mode_specs
+
+
+class Block:
+    def load(self, loader):
+        loader.add_option(
+            "block_global",
+            bool,
+            True,
+            """
+            Block connections from public IP addresses.
+            """,
+        )
+        loader.add_option(
+            "block_private",
+            bool,
+            False,
+            """
+            Block connections from local (private) IP addresses.
+            This option does not affect loopback addresses (connections from the local machine),
+            which are always permitted.
+            """,
+        )
+
+    def client_connected(self, client):
+        parts = client.peername[0].rsplit("%", 1)
+        address = ipaddress.ip_address(parts[0])
+        if isinstance(address, ipaddress.IPv6Address):
+            address = address.ipv4_mapped or address
+
+        if address.is_loopback or isinstance(client.proxy_mode, mode_specs.LocalMode):
+            return
+
+        if ctx.options.block_private and address.is_private:
+            logging.warning(
+                f"Client connection from {client.peername[0]} killed by block_private option."
+            )
+            client.error = "Connection killed by block_private."
+
+        if ctx.options.block_global and address.is_global:
+            logging.warning(
+                f"Client connection from {client.peername[0]} killed by block_global option."
+            )
+            client.error = "Connection killed by block_global."

xenproxy/addons/blocklist.py

@@ -0,0 +1,81 @@
+from collections.abc import Sequence
+from typing import NamedTuple
+
+from xenproxy import ctx
+from xenproxy import exceptions
+from xenproxy import flowfilter
+from xenproxy import http
+from xenproxy import version
+from xenproxy.net.http.status_codes import NO_RESPONSE
+
+
+class BlockSpec(NamedTuple):
+    matches: flowfilter.TFilter
+    status_code: int
+
+
+def parse_spec(option: str) -> BlockSpec:
+    """
+    Parses strings in the following format, enforces number of segments:
+
+        /flow-filter/status
+
+    """
+    sep, rem = option[0], option[1:]
+
+    parts = rem.split(sep, 2)
+    if len(parts) != 2:
+        raise ValueError("Invalid number of parameters (2 are expected)")
+    flow_patt, status = parts
+    try:
+        status_code = int(status)
+    except ValueError:
+        raise ValueError(f"Invalid HTTP status code: {status}")
+    flow_filter = flowfilter.parse(flow_patt)
+
+    return BlockSpec(matches=flow_filter, status_code=status_code)
+
+
+class BlockList:
+    def __init__(self) -> None:
+        self.items: list[BlockSpec] = []
+
+    def load(self, loader):
+        loader.add_option(
+            "block_list",
+            Sequence[str],
+            [],
+            """
+            Block matching requests and return an empty response with the specified HTTP status.
+            Option syntax is "/flow-filter/status-code", where flow-filter describes
+            which requests this rule should be applied to and status-code is the HTTP status code to return for
+            blocked requests. The separator ("/" in the example) can be any character.
+            Setting a non-standard status code of 444 will close the connection without sending a response.
+            """,
+        )
+
+    def configure(self, updated):
+        if "block_list" in updated:
+            self.items = []
+            for option in ctx.options.block_list:
+                try:
+                    spec = parse_spec(option)
+                except ValueError as e:
+                    raise exceptions.OptionsError(
+                        f"Cannot parse block_list option {option}: {e}"
+                    ) from e
+                self.items.append(spec)
+
+    def request(self, flow: http.HTTPFlow) -> None:
+        if flow.response or flow.error or not flow.live:
+            return
+
+        for spec in self.items:
+            if spec.matches(flow):
+                flow.metadata["blocklisted"] = True
+                if spec.status_code == NO_RESPONSE:
+                    flow.kill()
+                else:
+                    flow.response = http.Response.make(
+                        spec.status_code, headers={"Server": version.SERVER}
+                    )