bas-http 1.0.0__py3-none-any.whl

bas/cookies.py

@@ -0,0 +1,643 @@
+"""
+RFC 6265 compliant CookieJar.
+
+This module fixes the major cookie issues in curlcffi:
+  1. Cookies lost during redirects (#55)
+  2. Cookies not accumulating across session requests (#582)
+  3. Need for manual cookie header management
+  4. Improper domain/path matching
+
+Key design decisions:
+  - Pure Python cookie parsing (no dependency on curl's internal cookie engine)
+  - Per-domain storage with RFC 6265 domain matching
+  - Thread-safe with threading.Lock
+  - Cookie expiration handled correctly
+  - Secure, HttpOnly, SameSite attribute support
+  - Netscape file format import/export for compatibility
+"""
+
+import json
+import re
+import threading
+import time
+from datetime import datetime, timezone
+from http.cookiejar import CookieJar as _stdlib_jar
+from typing import Optional
+from urllib.parse import urlparse, urlunparse, quote, unquote
+
+
+class Cookie:
+    """
+    Represents a single HTTP cookie.
+
+    Attributes mirror the key properties in RFC 6265 Section 5.4.
+    """
+
+    __slots__ = (
+        "name", "value", "domain", "path", "expires",
+        "max_age", "secure", "http_only", "same_site",
+        "creation_time", "port", "port_specified",
+        "domain_specified", "domain_initial_dot",
+    )
+
+    def __init__(
+        self,
+        name: str,
+        value: str,
+        domain: str = "",
+        path: str = "/",
+        expires: Optional[float] = None,
+        max_age: Optional[int] = None,
+        secure: bool = False,
+        http_only: bool = False,
+        same_site: Optional[str] = None,
+        port: Optional[str] = None,
+        port_specified: bool = False,
+        domain_specified: bool = False,
+        domain_initial_dot: bool = False,
+    ):
+        self.name = name
+        self.value = value
+        self.domain_initial_dot = domain.startswith(".") or domain_initial_dot
+        self.domain = domain.lower().strip(".")
+        self.path = path or "/"
+        self.expires = expires
+        self.max_age = max_age
+        self.secure = secure
+        self.http_only = http_only
+        self.same_site = same_site
+        self.creation_time = time.time()
+        self.port = port
+        self.port_specified = port_specified
+        self.domain_specified = domain_specified
+
+    @property
+    def effective_expires(self) -> Optional[float]:
+        """Compute actual expiry time considering max_age vs expires."""
+        if self.max_age is not None:
+            return self.creation_time + self.max_age
+        return self.expires
+
+    @property
+    def is_expired(self) -> bool:
+        """Check if the cookie has expired."""
+        exp = self.effective_expires
+        if exp is None:
+            return False
+        return time.time() > exp
+
+    def matches_domain(self, request_domain: str) -> bool:
+        """
+        RFC 6265 Section 5.1.3 — Domain Matching.
+
+        A request-domain D matches a cookie-domain C if:
+          - The string is identical to C, OR
+          - The string is a rightmost match of C (suffix match)
+            AND C's first character is '.', OR
+          - The string is a rightmost match of C
+            AND C does NOT start with '.'
+            AND the request-domain is an IP address.
+        """
+        rd = request_domain.lower()
+        cd = self.domain
+
+        if not cd:
+            return False
+
+        # Exact match
+        if rd == cd:
+            return True
+
+        # If cookie domain starts with '.', it must be a suffix match
+        if self.domain_initial_dot:
+            return rd.endswith(cd) and rd[-len(cd) - 1] == "."
+
+        # If cookie domain doesn't start with '.', it must be an exact or
+        # suffix match where the request domain ends with the cookie domain
+        # and the character before is a '.' (to prevent "evil.com" matching "e.com")
+        if rd.endswith(cd):
+            if len(rd) == len(cd):
+                return True  # exact match
+            return rd[-(len(cd) + 1)] == "."
+
+        return False
+
+    def matches_path(self, request_path: str) -> bool:
+        """
+        RFC 6265 Section 5.1.4 — Path Matching.
+
+        A request-path P matches cookie-path C if:
+          - The string is identical to C, OR
+          - The string starts with C and the last character of C is '/', OR
+          - The string starts with C, the next character in the string
+            is either '/' or end-of-string.
+        """
+        cp = self.path
+        rp = request_path
+
+        if rp == cp:
+            return True
+
+        if cp == "/":
+            return True  # "/" matches everything
+
+        if rp.startswith(cp):
+            if len(rp) == len(cp):
+                return True  # exact match
+            if cp.endswith("/"):
+                return True
+            return rp[len(cp)] in ("/", "?", "#", "")
+
+        return False
+
+    def to_header(self) -> str:
+        """Serialize cookie as a Cookie header value."""
+        return f"{self.name}={self.value}"
+
+    def to_set_cookie_header(self) -> str:
+        """Serialize as a Set-Cookie header."""
+        parts = [f"{self.name}={self.value}"]
+        if self.domain:
+            parts.append(f"Domain={self.domain}")
+        if self.path:
+            parts.append(f"Path={self.path}")
+        if self.expires is not None:
+            parts.append(f"Expires={_format_cookie_time(self.expires)}")
+        if self.max_age is not None:
+            parts.append(f"Max-Age={self.max_age}")
+        if self.secure:
+            parts.append("Secure")
+        if self.http_only:
+            parts.append("HttpOnly")
+        if self.same_site:
+            parts.append(f"SameSite={self.same_site}")
+        return "; ".join(parts)
+
+    def to_dict(self) -> dict:
+        """Serialize cookie to a JSON-safe dict."""
+        return {
+            "name": self.name,
+            "value": self.value,
+            "domain": self.domain,
+            "path": self.path,
+            "expires": self.expires,
+            "max_age": self.max_age,
+            "secure": self.secure,
+            "http_only": self.http_only,
+            "same_site": self.same_site,
+            "creation_time": self.creation_time,
+            "port": self.port,
+            "port_specified": self.port_specified,
+            "domain_specified": self.domain_specified,
+            "domain_initial_dot": self.domain_initial_dot,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "Cookie":
+        """Deserialize a cookie from a dict."""
+        # creation_time is set in __init__, extract it separately
+        creation_time = data.pop("creation_time", None)
+        c = cls(**data)
+        if creation_time is not None:
+            c.creation_time = creation_time
+        return c
+
+    def __repr__(self) -> str:
+        return (
+            f"Cookie(name={self.name!r}, value={self.value!r}, "
+            f"domain={self.domain!r}, path={self.path!r}, "
+            f"secure={self.secure}, http_only={self.http_only})"
+        )
+
+    def __eq__(self, other) -> bool:
+        if not isinstance(other, Cookie):
+            return NotImplemented
+        return (
+            self.name == other.name
+            and self.domain == other.domain
+            and self.path == other.path
+        )
+
+    def __hash__(self) -> int:
+        return hash((self.name, self.domain, self.path))
+
+
+class CookieJar:
+    """
+    Thread-safe, RFC 6265 compliant cookie jar.
+
+    This is the heart of bas's cookie handling. Unlike curlcffi's
+    approach (which relies on libcurl's internal cookie engine and loses
+    cookies during redirects), bas manages cookies entirely in Python.
+
+    This means:
+      - Cookies accumulate correctly across ALL requests in a session
+      - Cookies are inherited during redirects (even cross-domain)
+      - No manual cookie header construction needed
+      - Full control over cookie serialization and persistence
+    """
+
+    def __init__(self):
+        self._cookies: list[Cookie] = []
+        self._lock = threading.Lock()
+
+    def __len__(self) -> int:
+        with self._lock:
+            return len(self._cookies)
+
+    def __iter__(self):
+        with self._lock:
+            return iter(list(self._cookies))
+
+    def __contains__(self, cookie: Cookie) -> bool:
+        with self._lock:
+            return cookie in self._cookies
+
+    def clear(self) -> None:
+        """Remove all cookies."""
+        with self._lock:
+            self._cookies.clear()
+
+    def add(self, cookie: Cookie) -> None:
+        """
+        Add a cookie to the jar, replacing any existing cookie
+        with the same (name, domain, path).
+        """
+        with self._lock:
+            # Remove existing cookie with same key
+            self._cookies = [
+                c for c in self._cookies
+                if not (c.name == cookie.name and c.domain == cookie.domain and c.path == cookie.path)
+            ]
+            if not cookie.is_expired:
+                self._cookies.append(cookie)
+
+    def remove(self, cookie: Cookie) -> None:
+        """Remove a specific cookie."""
+        with self._lock:
+            self._cookies = [
+                c for c in self._cookies
+                if not (c.name == cookie.name and c.domain == cookie.domain and c.path == cookie.path)
+            ]
+
+    def update(self, other: "CookieJar") -> None:
+        """Merge all cookies from another jar into this one."""
+        with self._lock:
+            for cookie in other:
+                self.add(cookie)
+
+    def get(
+        self,
+        name: str,
+        domain: Optional[str] = None,
+        path: Optional[str] = None,
+    ) -> Optional[Cookie]:
+        """Get a specific cookie by name, optionally filtered by domain/path."""
+        with self._lock:
+            for cookie in reversed(self._cookies):  # most recent first
+                if cookie.name == name:
+                    if domain and not cookie.matches_domain(domain):
+                        continue
+                    if path and not cookie.matches_path(path):
+                        continue
+                    if not cookie.is_expired:
+                        return cookie
+            return None
+
+    def get_all(
+        self,
+        name: Optional[str] = None,
+        domain: Optional[str] = None,
+        path: Optional[str] = None,
+    ) -> list[Cookie]:
+        """Get all matching cookies."""
+        with self._lock:
+            result = []
+            for cookie in self._cookies:
+                if cookie.is_expired:
+                    continue
+                if name and cookie.name != name:
+                    continue
+                if domain and not cookie.matches_domain(domain):
+                    continue
+                if path and not cookie.matches_path(path):
+                    continue
+                result.append(cookie)
+            return result
+
+    def match(self, url: str) -> list[Cookie]:
+        """
+        Get all cookies matching a given URL.
+
+        This is the primary method for determining which cookies to
+        send with a request. It performs full RFC 6265 matching.
+        """
+        parsed = urlparse(url)
+        domain = parsed.hostname or ""
+        path = parsed.path or "/"
+        is_secure = parsed.scheme == "https"
+
+        with self._lock:
+            result = []
+            expired_names = []
+
+            for cookie in self._cookies:
+                if cookie.is_expired:
+                    expired_names.append(cookie)
+                    continue
+
+                if not cookie.matches_domain(domain):
+                    continue
+                if not cookie.matches_path(path):
+                    continue
+                if cookie.secure and not is_secure:
+                    continue
+                result.append(cookie)
+
+            # Clean up expired cookies
+            for c in expired_names:
+                try:
+                    self._cookies.remove(c)
+                except ValueError:
+                    pass
+
+            return result
+
+    def parse_set_cookie(
+        self,
+        set_cookie_header: str,
+        request_url: str,
+    ) -> Optional[Cookie]:
+        """
+        Parse a Set-Cookie header and add the cookie to the jar.
+
+        This is where the magic happens — we parse the Set-Cookie header
+        ourselves instead of relying on curl's broken cookie engine.
+
+        Args:
+            set_cookie_header: The raw Set-Cookie header value
+            request_url: The URL that set this cookie (used for default domain/path)
+
+        Returns:
+            The parsed Cookie, or None if the cookie should be deleted
+        """
+        if not set_cookie_header:
+            return None
+
+        parsed_request = urlparse(request_url)
+        default_domain = parsed_request.hostname or ""
+        default_path = _guess_cookie_path(parsed_request.path or "/")
+
+        # Parse name=value and attributes
+        parts = [p.strip() for p in set_cookie_header.split(";")]
+        if not parts:
+            return None
+
+        # First part is name=value
+        name_value = parts[0]
+        eq_idx = name_value.find("=")
+        if eq_idx == -1:
+            name = unquote(name_value.strip())
+            value = ""
+        else:
+            name = unquote(name_value[:eq_idx].strip())
+            value = unquote(name_value[eq_idx + 1:].strip())
+
+        if not name:
+            return None
+
+        # Parse attributes
+        cookie = Cookie(
+            name=name,
+            value=value,
+            domain=default_domain,
+            path=default_path,
+        )
+
+        for part in parts[1:]:
+            part = part.strip()
+            eq_idx = part.find("=")
+
+            if eq_idx == -1:
+                attr_name = part.lower()
+                attr_value = ""
+            else:
+                attr_name = part[:eq_idx].strip().lower()
+                attr_value = part[eq_idx + 1:].strip()
+
+            if attr_name == "domain":
+                d = attr_value.lower().strip(".")
+                if d:
+                    cookie.domain = d
+                    cookie.domain_specified = True
+                    cookie.domain_initial_dot = attr_value.startswith(".")
+            elif attr_name == "path":
+                if attr_value and attr_value.startswith("/"):
+                    cookie.path = attr_value
+                else:
+                    cookie.path = default_path
+            elif attr_name == "expires":
+                try:
+                    cookie.expires = _parse_cookie_time(attr_value)
+                except (ValueError, TypeError):
+                    pass
+            elif attr_name == "max-age":
+                try:
+                    cookie.max_age = int(attr_value)
+                except (ValueError, TypeError):
+                    pass
+            elif attr_name == "secure":
+                cookie.secure = True
+            elif attr_name == "httponly":
+                cookie.http_only = True
+            elif attr_name == "samesite":
+                cookie.same_site = attr_value.capitalize()
+            elif attr_name == "port":
+                cookie.port = attr_value
+                cookie.port_specified = bool(attr_value)
+
+        # Add to jar
+        self.add(cookie)
+        return cookie
+
+    def parse_set_cookie_headers(
+        self,
+        headers: dict[str, str],
+        request_url: str,
+    ) -> list[Cookie]:
+        """
+        Parse ALL Set-Cookie headers from a response.
+
+        Handles both single-value and multi-value Set-Cookie headers.
+        """
+        cookies = []
+
+        # Handle both dict and list-of-tuples header formats
+        set_cookie_values = []
+        for key, value in headers.items():
+            if key.lower() == "set-cookie":
+                if isinstance(value, list):
+                    set_cookie_values.extend(value)
+                else:
+                    set_cookie_values.append(value)
+
+        for sc in set_cookie_values:
+            cookie = self.parse_set_cookie(sc, request_url)
+            if cookie:
+                cookies.append(cookie)
+
+        return cookies
+
+    def to_header(self, url: str) -> str:
+        """
+        Build a Cookie header value for the given URL.
+
+        This is what gets sent in the HTTP request. It returns ALL matching
+        cookies in the correct format, eliminating the need for users to
+        manually construct cookie headers.
+        """
+        cookies = self.match(url)
+        if not cookies:
+            return ""
+        return "; ".join(f"{c.name}={c.value}" for c in cookies)
+
+    def to_header_dict(self, url: str) -> dict[str, str]:
+        """Return a dict suitable for merging into request headers."""
+        header = self.to_header(url)
+        if header:
+            return {"Cookie": header}
+        return {}
+
+    def save_json(self, filepath: str) -> None:
+        """Save cookies to a JSON file."""
+        with self._lock:
+            data = [c.to_dict() for c in self._cookies if not c.is_expired]
+        with open(filepath, "w") as f:
+            json.dump(data, f, indent=2)
+
+    def load_json(self, filepath: str) -> None:
+        """Load cookies from a JSON file."""
+        try:
+            with open(filepath, "r") as f:
+                data = json.load(f)
+            with self._lock:
+                self._cookies.clear()
+                for item in data:
+                    c = Cookie.from_dict(item)
+                    if not c.is_expired:
+                        self._cookies.append(c)
+        except (FileNotFoundError, json.JSONDecodeError):
+            pass
+
+    def save_netscape(self, filepath: str) -> None:
+        """Save cookies in Netscape/Mozilla cookie file format."""
+        with self._lock:
+            lines = ["# Netscape HTTP Cookie File\n"]
+            for c in self._cookies:
+                if c.is_expired:
+                    continue
+                domain = ("." + c.domain) if not c.domain_initial_dot and not c.domain.startswith(".") else c.domain
+                flag = "TRUE" if c.domain_initial_dot else "FALSE"
+                secure = "TRUE" if c.secure else "FALSE"
+                expires = str(int(c.expires)) if c.expires else "0"
+                lines.append(
+                    f"{domain}\t{flag}\t{c.path}\t{secure}\t{expires}\t{c.name}\t{c.value}"
+                )
+        with open(filepath, "w") as f:
+            f.write("\n".join(lines) + "\n")
+
+    def load_netscape(self, filepath: str) -> None:
+        """Load cookies from a Netscape/Mozilla cookie file."""
+        try:
+            with open(filepath, "r") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+                    parts = line.split("\t")
+                    if len(parts) < 7:
+                        continue
+                    domain, _, path, secure, expires, name, value = parts[:7]
+                    initial_dot = domain.startswith(".")
+                    cookie = Cookie(
+                        name=name,
+                        value=value,
+                        domain=domain.strip("."),
+                        path=path,
+                        expires=float(expires) if expires != "0" else None,
+                        secure=secure.upper() == "TRUE",
+                        domain_initial_dot=initial_dot,
+                        domain_specified=True,
+                    )
+                    self.add(cookie)
+        except FileNotFoundError:
+            pass
+
+    def get_dict(self, url: str) -> dict[str, str]:
+        """Return cookies as a simple {name: value} dict for the given URL."""
+        return {c.name: c.value for c in self.match(url)}
+
+
+# ── Helper functions ─────────────────────────────────────────────────────────
+
+def _format_cookie_time(timestamp: float) -> str:
+    """Format a timestamp as an HTTP date string for Set-Cookie."""
+    dt = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+    return dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
+
+
+_COOKIE_MONTHS = {
+    "jan": 1, "feb": 2, "mar": 3, "apr": 4,
+    "may": 5, "jun": 6, "jul": 7, "aug": 8,
+    "sep": 9, "oct": 10, "nov": 11, "dec": 12,
+}
+
+
+def _parse_cookie_time(time_str: str) -> Optional[float]:
+    """
+    Parse various date formats found in Set-Cookie Expires headers.
+
+    Supports:
+      - RFC 1123: "Thu, 01 Dec 2025 16:00:00 GMT"
+      - RFC 1036: "Thursday, 01-Dec-25 16:00:00 GMT"
+      - Asctime: "Dec  1 16:00:00 2025 GMT"
+      - Simple: "Thu, 01 Dec 2025 16:00:00"
+    """
+    time_str = time_str.strip()
+
+    # Try common formats
+    formats = [
+        "%a, %d %b %Y %H:%M:%S %Z",
+        "%a, %d-%b-%Y %H:%M:%S %Z",
+        "%a, %d %b %Y %H:%M:%S",
+        "%A, %d-%b-%y %H:%M:%S %Z",
+        "%a %b %d %H:%M:%S %Y",
+        "%a %b  %d %H:%M:%S %Y",
+    ]
+
+    for fmt in formats:
+        try:
+            dt = datetime.strptime(time_str, fmt)
+            return dt.timestamp()
+        except ValueError:
+            continue
+
+    return None
+
+
+def _guess_cookie_path(request_path: str) -> str:
+    """
+    Guess a sensible default cookie path from the request path.
+
+    RFC 6265 Section 5.1.4 says the default path should be the
+    "longest directory path" — i.e., everything up to the last '/'.
+    """
+    if not request_path or not request_path.startswith("/"):
+        return "/"
+
+    # Find the last '/' that isn't the first character
+    last_slash = request_path.rfind("/")
+    if last_slash <= 0:
+        return "/"
+
+    return request_path[:last_slash]