python-urlopen 0.1.2__tar.gz → 0.1.4__tar.gz

{python_urlopen-0.1.2 → python_urlopen-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-urlopen
-Version: 0.1.2
+Version: 0.1.4
 Summary: Python urlopen wrapper.
 Home-page: https://github.com/ChenyangGao/python-modules/tree/main/python-urlopen
 License: MIT
@@ -20,16 +20,14 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Software Development
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Dist: brotli
-Requires-Dist: http_response (>=0.0.6)
+Requires-Dist: http_response (>=0.0.8)
 Requires-Dist: python-argtools (>=0.0.2)
-Requires-Dist: python-cookietools (>=0.0.8)
+Requires-Dist: python-cookietools (>=0.0.11)
 Requires-Dist: python-dicttools (>=0.0.2)
 Requires-Dist: python-filewrap (>=0.2.8)
-Requires-Dist: python-http_request (>=0.1.0)
+Requires-Dist: python-http_request (>=0.1.3)
 Requires-Dist: python-undefined (>=0.0.3)
 Requires-Dist: yarl
-Requires-Dist: zstandard
 Project-URL: Repository, https://github.com/ChenyangGao/python-modules/tree/main/python-urlopen
 Description-Content-Type: text/markdown
 

{python_urlopen-0.1.2 → python_urlopen-0.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-urlopen"
-version = "0.1.2"
+version = "0.1.4"
 description = "Python urlopen wrapper."
 authors = ["ChenyangGao <wosiwujm@gmail.com>"]
 license = "MIT"
@@ -27,16 +27,14 @@ include = [
 
 [tool.poetry.dependencies]
 python = "^3.12"
-brotli = "*"
-http_response = ">=0.0.6"
+http_response = ">=0.0.8"
 python-argtools = ">=0.0.2"
-python-cookietools = ">=0.0.8"
+python-cookietools = ">=0.0.11"
 python-dicttools = ">=0.0.2"
 python-filewrap = ">=0.2.8"
-python-http_request = ">=0.1.0"
+python-http_request = ">=0.1.3"
 python-undefined = ">=0.0.3"
 yarl = "*"
-zstandard = "*"
 
 [tool.poetry.scripts]
 python-urlopen = "urlopen.__main__:main"

{python_urlopen-0.1.2 → python_urlopen-0.1.4}/urlopen/__init__.py

@@ -2,105 +2,257 @@
 # coding: utf-8
 
 __author__ = "ChenyangGao <https://chenyanggao.github.io>"
-__version__ = (0, 1, 2)
+__version__ = (0, 1, 4)
 __all__ = ["urlopen", "request", "download"]
 
-import errno
-
-from collections import UserString
+from collections import defaultdict, deque, UserString
 from collections.abc import Buffer, Callable, Generator, Iterable, Mapping
 from copy import copy
-from gzip import decompress as decompress_gzip
-from http.client import HTTPResponse
+from http.client import HTTPConnection, HTTPSConnection, HTTPResponse
 from http.cookiejar import CookieJar
+from http.cookies import SimpleCookie
 from inspect import isgenerator
 from os import fsdecode, fstat, makedirs, PathLike
 from os.path import abspath, dirname, isdir, join as joinpath
 from shutil import COPY_BUFSIZE # type: ignore
-from socket import getdefaulttimeout, setdefaulttimeout
+from socket import socket
 from ssl import SSLContext, _create_unverified_context
 from types import EllipsisType
 from typing import cast, overload, Any, Literal
-from urllib.error import HTTPError
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlsplit, ParseResult, SplitResult
 from urllib.request import (
-    build_opener, BaseHandler, HTTPCookieProcessor, HTTPSHandler, 
-    HTTPRedirectHandler, OpenerDirector, Request, 
+    build_opener, AbstractHTTPHandler, BaseHandler, HTTPHandler, 
+    HTTPSHandler, HTTPRedirectHandler, OpenerDirector, Request, 
 )
-from zlib import compressobj, DEF_MEM_LEVEL, DEFLATED, MAX_WBITS
 
 from argtools import argcount
-from cookietools import cookies_dict_to_str
+from cookietools import cookies_to_str, extract_cookies, update_cookies
 from dicttools import iter_items
 from filewrap import bio_skip_iter, bio_chunk_iter, SupportsRead, SupportsWrite
 from http_request import normalize_request_args, SupportsGeturl
 from http_response import (
-    get_filename, get_length, is_chunked, is_range_request, 
+    decompress_response, get_filename, get_length, is_chunked, is_range_request, 
     parse_response, 
 )
+from property import locked_cacheproperty
 from yarl import URL
 from undefined import undefined, Undefined
 
 
 type string = Buffer | str | UserString
 
+if "__del__" not in HTTPConnection.__dict__:
+    setattr(HTTPConnection, "__del__", HTTPConnection.close)
+if "__del__" not in HTTPSConnection.__dict__:
+    setattr(HTTPSConnection, "__del__", HTTPSConnection.close)
 if "__del__" not in HTTPResponse.__dict__:
     setattr(HTTPResponse, "__del__", HTTPResponse.close)
 if "__del__" not in OpenerDirector.__dict__:
     setattr(OpenerDirector, "__del__", OpenerDirector.close)
 
-_cookies = CookieJar()
-_opener: OpenerDirector = build_opener(HTTPSHandler(context=_create_unverified_context()), HTTPCookieProcessor(_cookies))
-setattr(_opener, "cookies", _cookies)
+def _close_conn(self, /):
+    fp = self.fp
+    self.fp = None
+    pool = getattr(self, "pool", None)
+    conn = getattr(self, "connection", None)
+    if pool and conn:
+        try:
+            pool.return_connection(conn)
+        except NameError:
+            pass
+    else:
+        fp.close()
+
+setattr(HTTPResponse, "_close_conn", _close_conn)
+
+
+def is_ipv6(host: str, /) -> bool:
+    from ipaddress import _BaseV6, AddressValueError
+    try:
+        _BaseV6._ip_int_from_string(host) # type: ignore
+        return True
+    except AddressValueError:
+        return False
 
 
-if getdefaulttimeout() is None:
-    setdefaulttimeout(60)
+class HTTPCookieProcessor(BaseHandler):
+
+    def __init__(
+        self, 
+        /, 
+        cookies: None | CookieJar | SimpleCookie = None, 
+    ):
+        if cookies is None:
+            cookies = CookieJar()
+        self.cookies = cookies
+
+    def http_request(self, request):
+        cookies = self.cookies
+        if cookies:
+            if isinstance(cookies, SimpleCookie):
+                cookies = update_cookies(CookieJar(), cookies)
+            cookies.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        extract_cookies(self.cookies, request.full_url, response) # type: ignore
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+
+class ConnectionPool:
+
+    def __init__(
+        self, 
+        /, 
+        pool: None | defaultdict[str, deque[HTTPConnection] | deque[HTTPSConnection]] = None, 
+    ):
+        if pool is None:
+            pool = defaultdict(deque)
+        self.pool = pool
+
+    def __del__(self, /):
+        for dq in self.pool.values():
+            for con in dq:
+                con.close()
+
+    def __repr__(self, /) -> str:
+        cls = type(self)
+        return f"{cls.__module__}.{cls.__qualname__}({self.pool!r})"
+
+    def get_connection(
+        self, 
+        /, 
+        url: str | ParseResult | SplitResult, 
+        timeout: None | float = None, 
+    ) -> HTTPConnection | HTTPSConnection:
+        if isinstance(url, str):
+            url = urlsplit(url)
+        assert url.scheme, "not a complete URL"
+        host = url.hostname or "localhost"
+        if is_ipv6(host):
+            host = f"[{host}]"
+        port = url.port or (443 if url.scheme == 'https' else 80)
+        origin = f"{url.scheme}://{host}:{port}"
+        dq = self.pool[origin]
+        while True:
+            try:
+                con = dq.popleft()
+            except IndexError:
+                break
+            sock = con.sock
+            if not sock or getattr(sock, "_closed"):
+                con.connect()
+            else:
+                sock.setblocking(False)
+                try:
+                    if socket.recv(sock, 1):
+                        con.connect()
+                except BlockingIOError:
+                    pass
+                finally:
+                    sock.setblocking(True)
+            con.timeout = timeout
+            return con
+        if url.scheme == "https":
+            return HTTPSConnection(url.hostname or "localhost", url.port, timeout=timeout)
+        else:
+            return HTTPConnection(url.hostname or "localhost", url.port, timeout=timeout)
+
+    def return_connection(
+        self, 
+        con: HTTPConnection | HTTPSConnection, 
+        /, 
+    ) -> str:
+        if isinstance(con, HTTPSConnection):
+            scheme = "https"
+        else:
+            scheme = "http"
+        host = con.host
+        if is_ipv6(host):
+            host = f"[{host}]"
+        origin = f"{scheme}://{host}:{con.port}"
+        self.pool[origin].append(con) # type: ignore
+        return origin
+
+
+class KeepAliveBaseHTTPHandler(AbstractHTTPHandler):
+
+    @locked_cacheproperty
+    def pool(self, /) -> ConnectionPool:
+        return ConnectionPool()
+
+    def do_open(self, /, http_class, req, **http_conn_args) -> HTTPResponse:
+        host = req.host
+        if not host:
+            raise URLError("no host given")
+        pool = self.pool
+        if issubclass(http_class, HTTPSHandler):
+            origin = "https://" + host
+        else:
+            origin = "http://" + host
+        h = pool.get_connection(origin, timeout=req.timeout)
+        h.set_debuglevel(self._debuglevel) # type: ignore
+        headers = dict(req.unredirected_hdrs)
+        headers.update({k: v for k, v in req.headers.items()
+                        if k not in headers})
+        headers.setdefault("connection", "keep-alive")
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+        try:
+            try:
+                h.request(req.get_method(), req.selector, req.data, headers,
+                          encode_chunked=req.has_header('Transfer-encoding'))
+            except OSError as err:
+                raise URLError(err)
+            r = h.getresponse()
+        except:
+            pool.return_connection(h)
+            raise
+        r.url = req.get_full_url()
+        r.msg = r.reason
+        if headers.get("connection") == "keep-alive":
+            setattr(r, "pool", pool)
+        setattr(r, "connection", h)
+        return r
+
+
+class KeepAliveHTTPHandler(HTTPHandler, KeepAliveBaseHTTPHandler):
+    pass
+
+
+class KeepAliveHTTPSHandler(HTTPSHandler, KeepAliveBaseHTTPHandler):
+    pass
+
+
+_pool = ConnectionPool()
+_http_handler = KeepAliveHTTPHandler()
+_http_handler.pool = _pool
+_https_handler = KeepAliveHTTPSHandler(context=_create_unverified_context())
+_https_handler.pool = _pool
+_cookies = CookieJar()
+_opener: OpenerDirector = build_opener(
+    _http_handler, 
+    _https_handler, 
+    HTTPCookieProcessor(_cookies), 
+)
+setattr(_opener, "cookies", _cookies)
 
 
 class NoRedirectHandler(HTTPRedirectHandler):
+
     def redirect_request(self, /, *args, **kwds):
         return None
 
 
-def decompress_deflate(data: bytes, compresslevel: int = 9) -> bytes:
-    # Fork from: https://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations#answer-1089787
-    compress = compressobj(
-            compresslevel,  # level: 0-9
-            DEFLATED,       # method: must be DEFLATED
-            -MAX_WBITS,     # window size in bits:
-                            #   -15..-8: negate, suppress header
-                            #   8..15: normal
-                            #   16..30: subtract 16, gzip header
-            DEF_MEM_LEVEL,  # mem level: 1..8/9
-            0               # strategy:
-                            #   0 = Z_DEFAULT_STRATEGY
-                            #   1 = Z_FILTERED
-                            #   2 = Z_HUFFMAN_ONLY
-                            #   3 = Z_RLE
-                            #   4 = Z_FIXED
-    )
-    deflated = compress.compress(data)
-    deflated += compress.flush()
-    return deflated
-
-
-def decompress_response(response: HTTPResponse, /) -> bytes:
-    data = response.read()
-    content_encoding = response.headers.get("content-encoding")
-    match content_encoding:
-        case "gzip":
-            data = decompress_gzip(data)
-        case "deflate":
-            data = decompress_deflate(data)
-        case "br":
-            from brotli import decompress as decompress_br # type: ignore
-            data = decompress_br(data)
-        case "zstd":
-            from zstandard import decompress as decompress_zstd
-            data = decompress_zstd(data)
-    return data
-
-
 def urlopen(
     url: string | SupportsGeturl | URL | Request, 
     method: string = "GET", 
@@ -112,7 +264,7 @@ def urlopen(
     follow_redirects: bool = True, 
     proxies: None | Mapping[str, str] | Iterable[tuple[str, str]] = None, 
     context: None | SSLContext = None, 
-    cookies: None | CookieJar = None, 
+    cookies: None | CookieJar | SimpleCookie = None, 
     timeout: None | Undefined | float = undefined, 
     opener: None | OpenerDirector = _opener, 
     **_, 
@@ -140,18 +292,51 @@ def urlopen(
     headers_ = request.headers
     if opener is None:
         handlers: list[BaseHandler] = []
+        if cookies is None:
+            cookies = CookieJar()
     else:
         handlers = list(map(copy, getattr(opener, "handlers")))
         if cookies is None:
             cookies = getattr(opener, "cookies", None)
     if cookies and "cookie" not in headers_:
-        headers_["cookie"] = cookies_dict_to_str(cookies)
-    if context is not None:
-        handlers.append(HTTPSHandler(context=context))
-    elif opener is None:
-        handlers.append(HTTPSHandler(context=_create_unverified_context()))
-    if cookies is not None and (opener is None or all(
-        h.cookiejar is not cookies 
+        headers_["cookie"] = cookies_to_str(cookies)
+    if context is None:
+        if opener is None:
+            handlers.append(copy(_https_handler))
+        else:
+            for i, handler in enumerate(handlers):
+                if isinstance(handler, KeepAliveHTTPSHandler):
+                    break
+                elif isinstance(handler, HTTPSHandler):
+                    handlers[i] = copy(_https_handler)
+                    break
+            else:
+                handlers.append(copy(_https_handler))
+    else:
+        https_handler = KeepAliveHTTPSHandler(context=context)
+        https_handler.pool = _pool
+        if opener is None:
+            handlers.append(https_handler)
+        else:
+            for i, handler in enumerate(handlers):
+                if isinstance(handler, HTTPSHandler):
+                    handlers[i] = https_handler
+                    break
+            else:
+                handlers.append(https_handler)
+    if opener is None:
+        handlers.append(copy(_http_handler))
+    else:
+        for i, handler in enumerate(handlers):
+            if isinstance(handler, KeepAliveHTTPHandler):
+                break
+            elif isinstance(handler, HTTPHandler):
+                handlers[i] = copy(_http_handler)
+                break
+        else:
+            handlers.append(copy(_http_handler))
+    if cookies and (opener is None or all(
+        h.cookies is not cookies 
         for h in getattr(opener, "handlers") if isinstance(h, HTTPCookieProcessor)
     )):
         handlers.append(HTTPCookieProcessor(cookies))
@@ -167,7 +352,7 @@ def urlopen(
         if timeout is undefined:
             response = opener.open(request)
         else:
-            response = opener.open(request, timeout=cast(None|float, timeout))
+            response = opener.open(request, timeout=cast(None | float, timeout))
         setattr(response, "opener", opener)
         setattr(response, "cookies", response_cookies)
         return response
@@ -189,6 +374,7 @@ def request(
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
+    cookies: None | CookieJar | SimpleCookie = None, 
     *, 
     parse: None | EllipsisType = None, 
     **request_kwargs, 
@@ -205,6 +391,7 @@ def request(
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
+    cookies: None | CookieJar | SimpleCookie = None, 
     *, 
     parse: Literal[False], 
     **request_kwargs, 
@@ -221,6 +408,7 @@ def request(
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
+    cookies: None | CookieJar | SimpleCookie = None, 
     *, 
     parse: Literal[True], 
     **request_kwargs, 
@@ -237,6 +425,7 @@ def request[T](
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
+    cookies: None | CookieJar | SimpleCookie = None, 
     *, 
     parse: Callable[[HTTPResponse, bytes], T] | Callable[[HTTPResponse], T], 
     **request_kwargs, 
@@ -252,6 +441,7 @@ def request[T](
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
+    cookies: None | CookieJar | SimpleCookie = None, 
     *, 
     parse: None | EllipsisType| bool | Callable[[HTTPResponse, bytes], T] | Callable[[HTTPResponse], T] = None, 
     **request_kwargs, 
@@ -266,6 +456,7 @@ def request[T](
             files=files, 
             headers=headers, 
             follow_redirects=follow_redirects, 
+            cookies=cookies, 
             **request_kwargs, 
         )
     except HTTPError as e:
@@ -279,7 +470,7 @@ def request[T](
         return response
     with response:
         if isinstance(parse, bool):
-            data = decompress_response(response)
+            data = decompress_response(response.read(), response)
             if parse:
                 return parse_response(response, data)
             return data
@@ -287,8 +478,8 @@ def request[T](
         if ac == 1:
             return cast(Callable[[HTTPResponse], T], parse)(response)
         else:
-            return cast(Callable[[HTTPResponse, bytes], T], parse)(
-                response, decompress_response(response))
+            data = decompress_response(response.read(), response)
+            return cast(Callable[[HTTPResponse, bytes], T], parse)(response, data)
 
 
 def download(
@@ -373,7 +564,7 @@ def download(
                     return file
             elif content_length is not None and filesize > content_length:
                 raise OSError(
-                    errno.EIO, 
+                    5, # errno.EIO
                     f"file {file!r} is larger than url {url!r}: {filesize} > {content_length} (in bytes)", 
                 )
     reporthook_close: None | Callable = None
@@ -394,7 +585,10 @@ def download(
                 response.close()
                 response = urlopen(url, headers={**headers, "Range": "bytes=%d-" % filesize}, **request_kwargs)
                 if not is_range_request(response):
-                    raise OSError(errno.EIO, f"range request failed: {url!r}")
+                    raise OSError(
+                        5, # errno.EIO
+                        f"range request failed: {url!r}", 
+                    )
                 if reporthook is not None:
                     reporthook(filesize)
             elif resume: