http_client_request 0.0.6__tar.gz → 0.0.7__tar.gz

{http_client_request-0.0.6 → http_client_request-0.0.7}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: http_client_request
-Version: 0.0.6
-Summary: http.client request extension.
+Version: 0.0.7
+Summary: ttp.client request extension.
 Home-page: https://github.com/ChenyangGao/python-modules/tree/main/http_client_request
 License: MIT
 Keywords: http.client,request
@@ -26,6 +26,7 @@ Requires-Dist: python-cookietools (>=0.1.2)
 Requires-Dist: python-dicttools (>=0.0.4)
 Requires-Dist: python-filewrap (>=0.2.8)
 Requires-Dist: python-http_request (>=0.1.4)
+Requires-Dist: python-property (>=0.0.3)
 Requires-Dist: python-undefined (>=0.0.3)
 Requires-Dist: urllib3
 Requires-Dist: yarl

{http_client_request-0.0.6 → http_client_request-0.0.7}/http_client_request/__init__.py

@@ -4,9 +4,15 @@
 from __future__ import annotations
 
 __author__ = "ChenyangGao <https://chenyanggao.github.io>"
-__version__ = (0, 0, 6)
-__all__ = ["HTTPConnection", "HTTPSConnection", "HTTPResponse", "ConnectionPool", "request"]
+__version__ = (0, 0, 7)
+__all__ = [
+    "CONNECTION_POOL", "HTTPConnection", "HTTPSConnection", "HTTPResponse", 
+    "ConnectionPool", "request", 
+]
 
+import socket
+
+from array import array
 from collections import defaultdict, deque, UserString
 from collections.abc import Buffer, Callable, Iterable, Mapping
 from http.client import (
@@ -15,9 +21,11 @@ from http.client import (
 )
 from http.cookiejar import CookieJar
 from http.cookies import BaseCookie
+from io import BufferedReader
 from inspect import signature
 from os import PathLike
-from socket import socket
+from select import select
+from socket import socket as Socket
 from types import EllipsisType
 from typing import cast, overload, Any, Final, Literal
 from urllib.error import HTTPError
@@ -30,6 +38,7 @@ from dicttools import get_all_items
 from filewrap import SupportsRead
 from http_request import normalize_request_args, SupportsGeturl
 from http_response import decompress_response, parse_response
+from property import funcproperty
 from urllib3 import HTTPResponse as Urllib3HTTPResponse, HTTPHeaderDict
 from undefined import undefined, Undefined
 from yarl import URL
@@ -41,15 +50,100 @@ HTTP_CONNECTION_KWARGS: Final = signature(BaseHTTPConnection).parameters.keys()
 HTTPS_CONNECTION_KWARGS: Final = signature(BaseHTTPSConnection).parameters.keys()
 
 
+def get_host_pair(url: None | str, /) -> None | tuple[str, None | int]:
+    if not url:
+        return None
+    if not url.startswith(("http://", "https://")):
+        url = "http://" + url
+    urlp = urlsplit(url)
+    return urlp.hostname or "localhost", urlp.port
+
+
+def is_ipv6(host: str, /) -> bool:
+    from ipaddress import _BaseV6, AddressValueError
+    try:
+        _BaseV6._ip_int_from_string(host) # type: ignore
+        return True
+    except AddressValueError:
+        return False
+
+
+def sock_buf_readable(sock: Socket, /) -> bool:
+    rlist, *_ = select([sock], (), (), 0)
+    return bool(rlist)
+
+
+try:
+    from fcntl import ioctl
+    from termios import FIONREAD
+
+    def sock_bufsize(sock, /) -> int:
+        sock_size = array("i", [0])
+        ioctl(sock, FIONREAD, sock_size)
+        return sock_size[0]
+except ImportError:
+    from ctypes import byref, c_ulong, WinDLL # type: ignore
+    ws2_32 = WinDLL("ws2_32")
+    def sock_bufsize(sock, /) -> int:
+        FIONREAD = 0x4004667f
+        b = c_ulong(0)
+        ws2_32.ioctlsocket(sock.fileno(), FIONREAD, byref(b))
+        return b.value
+
+
 class HTTPResponse(BaseHTTPResponse):
+    _pos: int = 0
+    method: str
     pool: None | ConnectionPool = None
     connection: None | HTTPConnection | HTTPSConnection = None
 
     def __del__(self, /):
         self.close()
 
+    @funcproperty
+    def _fp(self, /) -> BufferedReader:
+        return self.fp
+
+    @property
+    def buffer_size(self, /) -> int:
+        fp = self._fp
+        sock = fp.raw._sock
+        cache_size = len(fp.peek())
+        while True:
+            buffer_size = cache_size + sock_bufsize(sock)
+            if cache_size == (cache_size := len(fp.peek())):
+                return buffer_size
+
+    @property
+    def unbuffer_size(self, /) -> int:
+        method = self.__dict__.get("method", "").upper()
+        content_length = self.length
+        if method == "HEAD" or content_length == 0:
+            return 0
+        if content_length:
+            return content_length - self.tell() - self.buffer_size
+        sock = self._fp.raw._sock
+        if sock_bufsize(sock) == sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF):
+            return -1
+        else:
+            return 0
+
+    @property
+    def unread_size(self, /) -> int:
+        method = self.__dict__.get("method", "").upper()
+        content_length = self.length
+        if method == "HEAD" or content_length == 0:
+            return 0
+        sock = self._fp.raw._sock
+        if content_length:
+            return content_length - self.tell()
+        elif sock_bufsize(sock) == sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF):
+            return -1
+        else:
+            return self.buffer_size
+
     def _close_conn(self, /):
-        fp = self.fp
+        fp = self._fp = self.fp
         setattr(self, "fp", None)
         try:
             pool = self.pool
@@ -70,12 +164,44 @@ class HTTPResponse(BaseHTTPResponse):
             preload_content=False, 
             decode_content=True, 
             original_response=self, 
-            #pool=getattr(self, "pool", None), 
             msg=self.headers, 
             request_method=getattr(self, "method", None), 
             request_url=self.url, 
         )
 
+    def read(self, /, amt=None) -> bytes:
+        data = super().read(amt)
+        self._pos += len(data)
+        return data
+
+    def read1(self, /, n=-1) -> bytes:
+        data = super().read1(n)
+        self._pos += len(data)
+        return data
+
+    def readinto(self, /, b) -> int:
+        count = super().readinto(b)
+        self._pos += count
+        return count
+
+    def readinto1(self, buffer, /) -> int:
+        count = super().readinto1(buffer)
+        self._pos += count
+        return count
+
+    def readline(self, limit=-1) -> bytes:
+        data = super().readline(limit)
+        self._pos += len(data)
+        return data
+
+    def readlines(self, hint=-1, /) -> list[bytes]:
+        ls = super().readlines(hint)
+        self._pos += sum(map(len, ls))
+        return ls
+
+    def tell(self, /) -> int:
+        return self._pos
+
 
 class HTTPConnection(BaseHTTPConnection):
     response_class = HTTPResponse
@@ -83,9 +209,20 @@ class HTTPConnection(BaseHTTPConnection):
     def __del__(self, /):
         self.close()
 
+    @property
+    def response(self, /) -> None | HTTPResponse:
+        return self.__response # type: ignore
+
     def getresponse(self, /) -> HTTPResponse:
         return cast(HTTPResponse, super().getresponse())
 
+    def putrequest(self, /, *args, **kwds):
+        while True:
+            try:
+                return super().putrequest(*args, **kwds)
+            except (ConnectionResetError, BrokenPipeError):
+                self.connect()
+
 
 class HTTPSConnection(BaseHTTPSConnection):
     response_class = HTTPResponse
@@ -93,26 +230,19 @@ class HTTPSConnection(BaseHTTPSConnection):
     def __del__(self, /):
         self.close()
 
+    @property
+    def response(self, /) -> None | HTTPResponse:
+        return self.__response # type: ignore
+
     def getresponse(self, /) -> HTTPResponse:
         return cast(HTTPResponse, super().getresponse())
 
-
-def get_host_pair(url: None | str, /) -> None | tuple[str, None | int]:
-    if not url:
-        return None
-    if not url.startswith(("http://", "https://")):
-        url = "http://" + url
-    urlp = urlsplit(url)
-    return urlp.hostname or "localhost", urlp.port
-
-
-def is_ipv6(host: str, /) -> bool:
-    from ipaddress import _BaseV6, AddressValueError
-    try:
-        _BaseV6._ip_int_from_string(host) # type: ignore
-        return True
-    except AddressValueError:
-        return False
+    def putrequest(self, /, *args, **kwds):
+        while True:
+            try:
+                return super().putrequest(*args, **kwds)
+            except (ConnectionResetError, BrokenPipeError):
+                self.connect()
 
 
 class ConnectionPool:
@@ -155,21 +285,28 @@ class ConnectionPool:
                 con = dq.popleft()
             except IndexError:
                 break
+            con.timeout = timeout
             sock = con.sock
+            resp = con.response
             if not sock or getattr(sock, "_closed"):
                 con.connect()
+            elif resp and 0 <= resp.unbuffer_size <= 1024 * 1024:
+                if resp.unread_size:
+                    try:
+                        resp.read()
+                    except (ConnectionResetError, BrokenPipeError):
+                        con.connect()
             else:
                 sock.setblocking(False)
                 try:
-                    if socket.recv(sock, 1):
+                    if Socket.recv(sock, 1):
                         con.connect()
                 except BlockingIOError:
                     pass
-                except ConnectionResetError:
+                except (ConnectionResetError, BrokenPipeError):
                     con.connect()
                 finally:
                     sock.setblocking(True)
-            con.timeout = timeout
             return con
         if url.scheme == "https":
             return HTTPSConnection(url.hostname or "localhost", url.port, timeout=timeout)
@@ -286,7 +423,7 @@ def request[T](
     raise_for_status: bool = True, 
     cookies: None | CookieJar | BaseCookie = None, 
     proxies: None | str | dict[str, str] = None, 
-    pool: None | Undefined | ConnectionPool = undefined,  
+    pool: None | Undefined | ConnectionPool = undefined, 
     *, 
     parse: None | EllipsisType| bool | Callable[[HTTPResponse, bytes], T] | Callable[[HTTPResponse], T] = None, 
     **request_kwargs, 
@@ -350,10 +487,16 @@ def request[T](
             connection = HTTPSConnection(**dict(get_all_items(request_kwargs, *HTTPS_CONNECTION_KWARGS)))
             if http_proxy:
                 connection.set_tunnel(*http_proxy)
+            else:
+                setattr(connection, "_tunnel_host", None)
+                setattr(connection, "_tunnel_port", None)
         else:
             connection = HTTPConnection(**dict(get_all_items(request_kwargs, *HTTP_CONNECTION_KWARGS)))
             if https_proxy:
                 connection.set_tunnel(*https_proxy)
+            else:
+                setattr(connection, "_tunnel_host", None)
+                setattr(connection, "_tunnel_port", None)
         connection.request(
             method, 
             urlunsplit(urlp._replace(scheme="", netloc="")), 

{http_client_request-0.0.6 → http_client_request-0.0.7}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "http_client_request"
-version = "0.0.6"
-description = "http.client request extension."
+version = "0.0.7"
+description = "ttp.client request extension."
 authors = ["ChenyangGao <wosiwujm@gmail.com>"]
 license = "MIT"
 readme = "readme.md"
@@ -33,6 +33,7 @@ python-cookietools = ">=0.1.2"
 python-dicttools = ">=0.0.4"
 python-filewrap = ">=0.2.8"
 python-http_request = ">=0.1.4"
+python-property = ">=0.0.3"
 python-undefined = ">=0.0.3"
 urllib3 = "*"
 yarl = "*"