exp-mitmproxy 12.2.3.post1__py3-none-any.whl

mitmproxy/net/http/cookies.py

@@ -0,0 +1,387 @@
+import email.utils
+import re
+import time
+from collections.abc import Iterable
+
+from mitmproxy.coretypes import multidict
+
+"""
+A flexible module for cookie parsing and manipulation.
+
+This module differs from usual standards-compliant cookie modules in a number
+of ways. We try to be as permissive as possible, and to retain even mal-formed
+information. Duplicate cookies are preserved in parsing, and can be set in
+formatting. We do attempt to escape and quote values where needed, but will not
+reject data that violate the specs.
+
+Parsing accepts the formats in RFC6265 and partially RFC2109 and RFC2965. We
+also parse the comma-separated variant of Set-Cookie that allows multiple
+cookies to be set in a single header. Serialization follows RFC6265.
+
+    http://tools.ietf.org/html/rfc6265
+    http://tools.ietf.org/html/rfc2109
+    http://tools.ietf.org/html/rfc2965
+"""
+
+_cookie_params = {
+    "expires",
+    "path",
+    "comment",
+    "max-age",
+    "secure",
+    "httponly",
+    "version",
+}
+
+ESCAPE = re.compile(r"([\"\\])")
+
+
+class CookieAttrs(multidict.MultiDict):
+    @staticmethod
+    def _kconv(key):
+        return key.lower()
+
+    @staticmethod
+    def _reduce_values(values):
+        # See the StickyCookieTest for a weird cookie that only makes sense
+        # if we take the last part.
+        return values[-1]
+
+
+TSetCookie = tuple[str, str | None, CookieAttrs]
+TPairs = list[tuple[str, str | None]]
+
+
+def _read_until(s, start, term):
+    """
+    Read until one of the characters in term is reached.
+    """
+    if start == len(s):
+        return "", start + 1
+    for i in range(start, len(s)):
+        if s[i] in term:
+            return s[start:i], i
+    return s[start : i + 1], i + 1
+
+
+def _read_quoted_string(s, start):
+    """
+    start: offset to the first quote of the string to be read
+
+    A sort of loose super-set of the various quoted string specifications.
+
+    RFC6265 disallows backslashes or double quotes within quoted strings.
+    Prior RFCs use backslashes to escape. This leaves us free to apply
+    backslash escaping by default and be compatible with everything.
+    """
+    escaping = False
+    ret = []
+    # Skip the first quote
+    i = start  # initialize in case the loop doesn't run.
+    for i in range(start + 1, len(s)):
+        if escaping:
+            ret.append(s[i])
+            escaping = False
+        elif s[i] == '"':
+            break
+        elif s[i] == "\\":
+            escaping = True
+        else:
+            ret.append(s[i])
+    return "".join(ret), i + 1
+
+
+def _read_key(s, start, delims=";="):
+    """
+    Read a key - the LHS of a token/value pair in a cookie.
+    """
+    return _read_until(s, start, delims)
+
+
+def _read_value(s, start, delims):
+    """
+    Reads a value - the RHS of a token/value pair in a cookie.
+    """
+    if start >= len(s):
+        return "", start
+    elif s[start] == '"':
+        return _read_quoted_string(s, start)
+    else:
+        return _read_until(s, start, delims)
+
+
+def _read_cookie_pairs(s, off=0):
+    """
+    Read pairs of lhs=rhs values from Cookie headers.
+
+    off: start offset
+    """
+    pairs = []
+
+    while True:
+        lhs, off = _read_key(s, off)
+        lhs = lhs.lstrip()
+
+        rhs = ""
+        if off < len(s) and s[off] == "=":
+            rhs, off = _read_value(s, off + 1, ";")
+        if rhs or lhs:
+            pairs.append([lhs, rhs])
+
+        off += 1
+
+        if not off < len(s):
+            break
+
+    return pairs, off
+
+
+def _read_set_cookie_pairs(s: str, off=0) -> tuple[list[TPairs], int]:
+    """
+    Read pairs of lhs=rhs values from SetCookie headers while handling multiple cookies.
+
+    off: start offset
+    specials: attributes that are treated specially
+    """
+    cookies: list[TPairs] = []
+    pairs: TPairs = []
+
+    while True:
+        lhs, off = _read_key(s, off, ";=,")
+        lhs = lhs.lstrip()
+
+        rhs = ""
+        if off < len(s) and s[off] == "=":
+            rhs, off = _read_value(s, off + 1, ";,")
+
+            # Special handling of attributes
+            if lhs.lower() == "expires":
+                # 'expires' values can contain commas in them so they need to
+                # be handled separately.
+
+                # We actually bank on the fact that the expires value WILL
+                # contain a comma. Things will fail, if they don't.
+
+                # '3' is just a heuristic we use to determine whether we've
+                # only read a part of the expires value and we should read more.
+                if len(rhs) <= 3:
+                    trail, off = _read_value(s, off + 1, ";,")
+                    rhs = rhs + "," + trail
+
+            # as long as there's a "=", we consider it a pair
+            pairs.append((lhs, rhs))
+
+        elif lhs:
+            pairs.append((lhs, None))
+
+        # comma marks the beginning of a new cookie
+        if off < len(s) and s[off] == ",":
+            cookies.append(pairs)
+            pairs = []
+
+        off += 1
+
+        if not off < len(s):
+            break
+
+    if pairs or not cookies:
+        cookies.append(pairs)
+
+    return cookies, off
+
+
+def _has_special(s: str) -> bool:
+    for i in s:
+        if i in '",;\\':
+            return True
+        o = ord(i)
+        if o < 0x21 or o > 0x7E:
+            return True
+    return False
+
+
+def _format_pairs(pairs, specials=(), sep="; "):
+    """
+    specials: A lower-cased list of keys that will not be quoted.
+    """
+    vals = []
+    for k, v in pairs:
+        if v is None:
+            val = k
+        elif k.lower() not in specials and _has_special(v):
+            v = ESCAPE.sub(r"\\\1", v)
+            v = '"%s"' % v
+            val = f"{k}={v}"
+        else:
+            val = f"{k}={v}"
+        vals.append(val)
+    return sep.join(vals)
+
+
+def _format_set_cookie_pairs(lst):
+    return _format_pairs(lst, specials=("expires", "path"))
+
+
+def parse_cookie_header(line):
+    """
+    Parse a Cookie header value.
+    Returns a list of (lhs, rhs) tuples.
+    """
+    pairs, off_ = _read_cookie_pairs(line)
+    return pairs
+
+
+def parse_cookie_headers(cookie_headers):
+    cookie_list = []
+    for header in cookie_headers:
+        cookie_list.extend(parse_cookie_header(header))
+    return cookie_list
+
+
+def format_cookie_header(lst):
+    """
+    Formats a Cookie header value.
+    """
+    return _format_pairs(lst)
+
+
+def parse_set_cookie_header(line: str) -> list[TSetCookie]:
+    """
+    Parse a Set-Cookie header value
+
+    Returns:
+        A list of (name, value, attrs) tuples, where attrs is a
+        CookieAttrs dict of attributes. No attempt is made to parse attribute
+        values - they are treated purely as strings.
+    """
+    cookie_pairs, off = _read_set_cookie_pairs(line)
+    cookies = []
+    for pairs in cookie_pairs:
+        if pairs:
+            cookie, *attrs = pairs
+            cookies.append((cookie[0], cookie[1], CookieAttrs(attrs)))
+    return cookies
+
+
+def parse_set_cookie_headers(headers: Iterable[str]) -> list[TSetCookie]:
+    rv = []
+    for header in headers:
+        cookies = parse_set_cookie_header(header)
+        rv.extend(cookies)
+    return rv
+
+
+def format_set_cookie_header(set_cookies: list[TSetCookie]) -> str:
+    """
+    Formats a Set-Cookie header value.
+    """
+
+    rv = []
+
+    for name, value, attrs in set_cookies:
+        pairs = [(name, value)]
+        pairs.extend(attrs.fields if hasattr(attrs, "fields") else attrs)
+
+        rv.append(_format_set_cookie_pairs(pairs))
+
+    return ", ".join(rv)
+
+
+def refresh_set_cookie_header(c: str, delta: int) -> str:
+    """
+    Args:
+        c: A Set-Cookie string
+        delta: Time delta in seconds
+    Returns:
+        A refreshed Set-Cookie string
+    Raises:
+        ValueError, if the cookie is invalid.
+    """
+    cookies = parse_set_cookie_header(c)
+    for cookie in cookies:
+        name, value, attrs = cookie
+        if not name or not value:
+            raise ValueError("Invalid Cookie")
+
+        if "expires" in attrs:
+            e = email.utils.parsedate_tz(attrs["expires"])
+            if e:
+                f = email.utils.mktime_tz(e) + delta
+                attrs.set_all("expires", [email.utils.formatdate(f, usegmt=True)])
+            else:
+                # This can happen when the expires tag is invalid.
+                # reddit.com sends a an expires tag like this: "Thu, 31 Dec
+                # 2037 23:59:59 GMT", which is valid RFC 1123, but not
+                # strictly correct according to the cookie spec. Browsers
+                # appear to parse this tolerantly - maybe we should too.
+                # For now, we just ignore this.
+                del attrs["expires"]
+    return format_set_cookie_header(cookies)
+
+
+def get_expiration_ts(cookie_attrs):
+    """
+    Determines the time when the cookie will be expired.
+
+    Considering both 'expires' and 'max-age' parameters.
+
+    Returns: timestamp of when the cookie will expire.
+             None, if no expiration time is set.
+    """
+    if "expires" in cookie_attrs:
+        e = email.utils.parsedate_tz(cookie_attrs["expires"])
+        if e:
+            return email.utils.mktime_tz(e)
+
+    elif "max-age" in cookie_attrs:
+        try:
+            max_age = int(cookie_attrs["Max-Age"])
+        except ValueError:
+            pass
+        else:
+            now_ts = time.time()
+            return now_ts + max_age
+
+    return None
+
+
+def is_expired(cookie_attrs):
+    """
+    Determines whether a cookie has expired.
+
+    Returns: boolean
+    """
+
+    exp_ts = get_expiration_ts(cookie_attrs)
+    now_ts = time.time()
+
+    # If no expiration information was provided with the cookie
+    if exp_ts is None:
+        return False
+    else:
+        return exp_ts <= now_ts
+
+
+def group_cookies(pairs):
+    """
+    Converts a list of pairs to a (name, value, attrs) for each cookie.
+    """
+
+    if not pairs:
+        return []
+
+    cookie_list = []
+
+    # First pair is always a new cookie
+    name, value = pairs[0]
+    attrs = []
+
+    for k, v in pairs[1:]:
+        if k.lower() in _cookie_params:
+            attrs.append((k, v))
+        else:
+            cookie_list.append((name, value, CookieAttrs(attrs)))
+            name, value, attrs = k, v, []
+
+    cookie_list.append((name, value, CookieAttrs(attrs)))
+    return cookie_list

mitmproxy/net/http/headers.py

@@ -0,0 +1,113 @@
+import collections
+import re
+
+
+def parse_content_type(c: str) -> tuple[str, str, dict[str, str]] | None:
+    """
+    A simple parser for content-type values. Returns a (type, subtype,
+    parameters) tuple, where type and subtype are strings, and parameters
+    is a dict. If the string could not be parsed, return None.
+
+    E.g. the following string:
+
+        text/html; charset=UTF-8
+
+    Returns:
+
+        ("text", "html", {"charset": "UTF-8"})
+    """
+    parts = c.split(";", 1)
+    ts = parts[0].split("/", 1)
+    if len(ts) != 2:
+        return None
+    d = collections.OrderedDict()
+    if len(parts) == 2:
+        for i in parts[1].split(";"):
+            clause = i.split("=", 1)
+            if len(clause) == 2:
+                d[clause[0].strip()] = clause[1].strip()
+    return ts[0].lower(), ts[1].lower(), d
+
+
+def assemble_content_type(type, subtype, parameters):
+    if not parameters:
+        return f"{type}/{subtype}"
+    params = "; ".join(f"{k}={v}" for k, v in parameters.items())
+    return f"{type}/{subtype}; {params}"
+
+
+def infer_content_encoding(content_type: str, content: bytes = b"") -> str:
+    """
+    Infer the encoding of content from the content-type header.
+    """
+    enc = None
+
+    # BOM has the highest priority
+    if content.startswith(b"\x00\x00\xfe\xff"):
+        enc = "utf-32be"
+    elif content.startswith(b"\xff\xfe\x00\x00"):
+        enc = "utf-32le"
+    elif content.startswith(b"\xfe\xff"):
+        enc = "utf-16be"
+    elif content.startswith(b"\xff\xfe"):
+        enc = "utf-16le"
+    elif content.startswith(b"\xef\xbb\xbf"):
+        # 'utf-8-sig' will strip the BOM on decode
+        enc = "utf-8-sig"
+    elif parsed_content_type := parse_content_type(content_type):
+        # Use the charset from the header if possible
+        enc = parsed_content_type[2].get("charset")
+
+    # Otherwise, infer the encoding
+    if not enc and "json" in content_type:
+        enc = "utf8"
+
+    if not enc and "html" in content_type:
+        meta_charset = re.search(
+            rb"""<meta[^>]+charset=['"]?([^'">]+)""", content, re.IGNORECASE
+        )
+        if meta_charset:
+            enc = meta_charset.group(1).decode("ascii", "ignore")
+        else:
+            # Fallback to utf8 for html
+            # Ref: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
+            # > 9. [snip] the comprehensive UTF-8 encoding is suggested.
+            enc = "utf8"
+
+    if not enc and "xml" in content_type:
+        if xml_encoding := re.search(
+            rb"""<\?xml[^\?>]+encoding=['"]([^'"\?>]+)""", content, re.IGNORECASE
+        ):
+            enc = xml_encoding.group(1).decode("ascii", "ignore")
+        else:
+            # Fallback to utf8 for xml
+            # Ref: https://datatracker.ietf.org/doc/html/rfc7303#section-8.5
+            # > the XML processor [snip] to determine an encoding of UTF-8.
+            enc = "utf8"
+
+    if not enc and ("javascript" in content_type or "ecmascript" in content_type):
+        # Fallback to utf8 for javascript
+        # Ref: https://datatracker.ietf.org/doc/html/rfc9239#section-4.2
+        # > 3. Else, the character encoding scheme is assumed to be UTF-8
+        enc = "utf8"
+
+    if not enc and "text/css" in content_type:
+        # @charset rule must be the very first thing.
+        css_charset = re.match(rb"""@charset "([^"]+)";""", content, re.IGNORECASE)
+        if css_charset:
+            enc = css_charset.group(1).decode("ascii", "ignore")
+        else:
+            # Fallback to utf8 for css
+            # Ref: https://drafts.csswg.org/css-syntax/#determine-the-fallback-encoding
+            # > 4. Otherwise, return utf-8
+            enc = "utf8"
+
+    # Fallback to latin-1
+    if not enc:
+        enc = "latin-1"
+
+    # Use GB 18030 as the superset of GB2312 and GBK to fix common encoding problems on Chinese websites.
+    if enc.lower() in ("gb2312", "gbk"):
+        enc = "gb18030"
+
+    return enc

mitmproxy/net/http/http1/__init__.py

@@ -0,0 +1,21 @@
+from .assemble import assemble_body
+from .assemble import assemble_request
+from .assemble import assemble_request_head
+from .assemble import assemble_response
+from .assemble import assemble_response_head
+from .read import connection_close
+from .read import expected_http_body_size
+from .read import read_request_head
+from .read import read_response_head
+
+__all__ = [
+    "read_request_head",
+    "read_response_head",
+    "connection_close",
+    "expected_http_body_size",
+    "assemble_request",
+    "assemble_request_head",
+    "assemble_response",
+    "assemble_response_head",
+    "assemble_body",
+]

mitmproxy/net/http/http1/assemble.py

@@ -0,0 +1,99 @@
+def assemble_request(request):
+    if request.data.content is None:
+        raise ValueError("Cannot assemble flow with missing content")
+    head = assemble_request_head(request)
+    body = b"".join(
+        assemble_body(
+            request.data.headers, [request.data.content], request.data.trailers
+        )
+    )
+    return head + body
+
+
+def assemble_request_head(request):
+    first_line = _assemble_request_line(request.data)
+    headers = _assemble_request_headers(request.data)
+    return b"%s\r\n%s\r\n" % (first_line, headers)
+
+
+def assemble_response(response):
+    if response.data.content is None:
+        raise ValueError("Cannot assemble flow with missing content")
+    head = assemble_response_head(response)
+    body = b"".join(
+        assemble_body(
+            response.data.headers, [response.data.content], response.data.trailers
+        )
+    )
+    return head + body
+
+
+def assemble_response_head(response):
+    first_line = _assemble_response_line(response.data)
+    headers = _assemble_response_headers(response.data)
+    return b"%s\r\n%s\r\n" % (first_line, headers)
+
+
+def assemble_body(headers, body_chunks, trailers):
+    if "chunked" in headers.get("transfer-encoding", "").lower():
+        for chunk in body_chunks:
+            if chunk:
+                yield b"%x\r\n%s\r\n" % (len(chunk), chunk)
+        if trailers:
+            yield b"0\r\n%s\r\n" % trailers
+        else:
+            yield b"0\r\n\r\n"
+    else:
+        if trailers:
+            raise ValueError(
+                "Sending HTTP/1.1 trailer headers requires transfer-encoding: chunked"
+            )
+        for chunk in body_chunks:
+            yield chunk
+
+
+def _assemble_request_line(request_data):
+    """
+    Args:
+        request_data (mitmproxy.net.http.request.RequestData)
+    """
+    if request_data.method.upper() == b"CONNECT":
+        return b"%s %s %s" % (
+            request_data.method,
+            request_data.authority,
+            request_data.http_version,
+        )
+    elif request_data.authority:
+        return b"%s %s://%s%s %s" % (
+            request_data.method,
+            request_data.scheme,
+            request_data.authority,
+            request_data.path,
+            request_data.http_version,
+        )
+    else:
+        return b"%s %s %s" % (
+            request_data.method,
+            request_data.path,
+            request_data.http_version,
+        )
+
+
+def _assemble_request_headers(request_data):
+    """
+    Args:
+        request_data (mitmproxy.net.http.request.RequestData)
+    """
+    return bytes(request_data.headers)
+
+
+def _assemble_response_line(response_data):
+    return b"%s %d %s" % (
+        response_data.http_version,
+        response_data.status_code,
+        response_data.reason,
+    )
+
+
+def _assemble_response_headers(response):
+    return bytes(response.headers)