python-urlopen 0.1.0__tar.gz → 0.1.2__tar.gz

{python_urlopen-0.1.0 → python_urlopen-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-urlopen
-Version: 0.1.0
+Version: 0.1.2
 Summary: Python urlopen wrapper.
 Home-page: https://github.com/ChenyangGao/python-modules/tree/main/python-urlopen
 License: MIT
@@ -21,12 +21,12 @@ Classifier: Topic :: Software Development
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: brotli
-Requires-Dist: http_response (>=0.0.5)
+Requires-Dist: http_response (>=0.0.6)
 Requires-Dist: python-argtools (>=0.0.2)
-Requires-Dist: python-dicttools (>=0.0.1)
-Requires-Dist: python-ensure (>=0.0.1)
+Requires-Dist: python-cookietools (>=0.0.8)
+Requires-Dist: python-dicttools (>=0.0.2)
 Requires-Dist: python-filewrap (>=0.2.8)
-Requires-Dist: python-http_request (>=0.0.9)
+Requires-Dist: python-http_request (>=0.1.0)
 Requires-Dist: python-undefined (>=0.0.3)
 Requires-Dist: yarl
 Requires-Dist: zstandard

{python_urlopen-0.1.0 → python_urlopen-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-urlopen"
-version = "0.1.0"
+version = "0.1.2"
 description = "Python urlopen wrapper."
 authors = ["ChenyangGao <wosiwujm@gmail.com>"]
 license = "MIT"
@@ -28,12 +28,12 @@ include = [
 [tool.poetry.dependencies]
 python = "^3.12"
 brotli = "*"
-http_response = ">=0.0.5"
+http_response = ">=0.0.6"
 python-argtools = ">=0.0.2"
-python-dicttools = ">=0.0.1"
-python-ensure = ">=0.0.1"
+python-cookietools = ">=0.0.8"
+python-dicttools = ">=0.0.2"
 python-filewrap = ">=0.2.8"
-python-http_request = ">=0.0.9"
+python-http_request = ">=0.1.0"
 python-undefined = ">=0.0.3"
 yarl = "*"
 zstandard = "*"

{python_urlopen-0.1.0 → python_urlopen-0.1.2}/urlopen/__init__.py

@@ -2,13 +2,14 @@
 # coding: utf-8
 
 __author__ = "ChenyangGao <https://chenyanggao.github.io>"
-__version__ = (0, 1, 0)
+__version__ = (0, 1, 2)
 __all__ = ["urlopen", "request", "download"]
 
 import errno
 
 from collections import UserString
-from collections.abc import Buffer, Callable, Generator, Iterable, Mapping, Sequence
+from collections.abc import Buffer, Callable, Generator, Iterable, Mapping
+from copy import copy
 from gzip import decompress as decompress_gzip
 from http.client import HTTPResponse
 from http.cookiejar import CookieJar
@@ -28,8 +29,8 @@ from urllib.request import (
 from zlib import compressobj, DEF_MEM_LEVEL, DEFLATED, MAX_WBITS
 
 from argtools import argcount
+from cookietools import cookies_dict_to_str
 from dicttools import iter_items
-from ensure import ensure_buffer
 from filewrap import bio_skip_iter, bio_chunk_iter, SupportsRead, SupportsWrite
 from http_request import normalize_request_args, SupportsGeturl
 from http_response import (
@@ -47,7 +48,9 @@ if "__del__" not in HTTPResponse.__dict__:
 if "__del__" not in OpenerDirector.__dict__:
     setattr(OpenerDirector, "__del__", OpenerDirector.close)
 
-_opener: OpenerDirector = build_opener(HTTPSHandler(context=_create_unverified_context()))
+_cookies = CookieJar()
+_opener: OpenerDirector = build_opener(HTTPSHandler(context=_create_unverified_context()), HTTPCookieProcessor(_cookies))
+setattr(_opener, "cookies", _cookies)
 
 
 if getdefaulttimeout() is None:
@@ -101,16 +104,17 @@ def decompress_response(response: HTTPResponse, /) -> bytes:
 def urlopen(
     url: string | SupportsGeturl | URL | Request, 
     method: string = "GET", 
-    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    params: None | string | Mapping | Iterable[tuple[Any, Any]] = None, 
     data: Any = None, 
     json: Any = None, 
+    files: None | Mapping[string, Any] | Iterable[tuple[string, Any]] = None, 
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     proxies: None | Mapping[str, str] | Iterable[tuple[str, str]] = None, 
     context: None | SSLContext = None, 
     cookies: None | CookieJar = None, 
     timeout: None | Undefined | float = undefined, 
-    opener: None | OpenerDirector = None, 
+    opener: None | OpenerDirector = _opener, 
     **_, 
 ) -> HTTPResponse:
     if isinstance(url, Request):
@@ -119,46 +123,69 @@ def urlopen(
         if isinstance(data, PathLike):
             data = bio_chunk_iter(open(data, "rb"))
         elif isinstance(data, SupportsRead):
-            data = map(ensure_buffer, bio_chunk_iter(data))
+            data = bio_chunk_iter(data)
         request = Request(**normalize_request_args( # type: ignore
             method=method, 
             url=url, 
             params=params, 
             data=data, 
             json=json, 
+            files=files, 
             headers=headers, 
             ensure_ascii=True, 
         ))
         if proxies:
             for host, type in iter_items(proxies):
                 request.set_proxy(host, type)
+    headers_ = request.headers
     if opener is None:
         handlers: list[BaseHandler] = []
-        if context is not None:
-            handlers.append(HTTPSHandler(context=context))
-        if cookies is not None:
-            handlers.append(HTTPCookieProcessor(cookies))
-        if not follow_redirects:
-            handlers.append(NoRedirectHandler())
-        if handlers:
-            if not isinstance(handlers[0], HTTPSHandler):
-                handlers.insert(0, HTTPSHandler(context=_create_unverified_context()))
-            opener = build_opener(*handlers)
-        else:
-            opener = _opener
-    if timeout is undefined:
-        return opener.open(request)
     else:
-        return opener.open(request, timeout=cast(None|float, timeout))
+        handlers = list(map(copy, getattr(opener, "handlers")))
+        if cookies is None:
+            cookies = getattr(opener, "cookies", None)
+    if cookies and "cookie" not in headers_:
+        headers_["cookie"] = cookies_dict_to_str(cookies)
+    if context is not None:
+        handlers.append(HTTPSHandler(context=context))
+    elif opener is None:
+        handlers.append(HTTPSHandler(context=_create_unverified_context()))
+    if cookies is not None and (opener is None or all(
+        h.cookiejar is not cookies 
+        for h in getattr(opener, "handlers") if isinstance(h, HTTPCookieProcessor)
+    )):
+        handlers.append(HTTPCookieProcessor(cookies))
+    response_cookies = CookieJar()
+    if cookies is None:
+        cookies = response_cookies
+    handlers.append(HTTPCookieProcessor(response_cookies))
+    if not follow_redirects:
+        handlers.append(NoRedirectHandler())
+    opener = build_opener(*handlers)
+    setattr(opener, "cookies", cookies)
+    try:
+        if timeout is undefined:
+            response = opener.open(request)
+        else:
+            response = opener.open(request, timeout=cast(None|float, timeout))
+        setattr(response, "opener", opener)
+        setattr(response, "cookies", response_cookies)
+        return response
+    except HTTPError as e:
+        if response := getattr(e, "file", None):
+            setattr(response, "opener", opener)
+            setattr(response, "cookies", response_cookies)
+        raise
 
 
 @overload
 def request(
     url: string | SupportsGeturl | URL | Request, 
     method: string = "GET", 
-    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    params: None | string | Mapping | Iterable[tuple[Any, Any]] = None, 
     data: Any = None, 
     json: Any = None, 
+    files: None | Mapping[string, Any] | Iterable[tuple[string, Any]] = None, 
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
@@ -171,9 +198,10 @@ def request(
 def request(
     url: string | SupportsGeturl | URL | Request, 
     method: string = "GET", 
-    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    params: None | string | Mapping | Iterable[tuple[Any, Any]] = None, 
     data: Any = None, 
     json: Any = None, 
+    files: None | Mapping[string, Any] | Iterable[tuple[string, Any]] = None, 
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
@@ -186,9 +214,10 @@ def request(
 def request(
     url: string | SupportsGeturl | URL | Request, 
     method: string = "GET", 
-    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    params: None | string | Mapping | Iterable[tuple[Any, Any]] = None, 
     data: Any = None, 
     json: Any = None, 
+    files: None | Mapping[string, Any] | Iterable[tuple[string, Any]] = None, 
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
@@ -201,9 +230,10 @@ def request(
 def request[T](
     url: string | SupportsGeturl | URL | Request, 
     method: string = "GET", 
-    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    params: None | string | Mapping | Iterable[tuple[Any, Any]] = None, 
     data: Any = None, 
     json: Any = None, 
+    files: None | Mapping[string, Any] | Iterable[tuple[string, Any]] = None, 
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
@@ -215,9 +245,10 @@ def request[T](
 def request[T](
     url: string | SupportsGeturl | URL | Request, 
     method: string = "GET", 
-    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    params: None | string | Mapping | Iterable[tuple[Any, Any]] = None, 
     data: Any = None, 
     json: Any = None, 
+    files: None | Mapping[string, Any] | Iterable[tuple[string, Any]] = None, 
     headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
     follow_redirects: bool = True, 
     raise_for_status: bool = True, 
@@ -232,6 +263,7 @@ def request[T](
             params=params, 
             data=data, 
             json=json, 
+            files=files, 
             headers=headers, 
             follow_redirects=follow_redirects, 
             **request_kwargs, 
@@ -310,12 +342,10 @@ def download(
         chunksize = COPY_BUFSIZE
     headers = request_kwargs["headers"] = dict(headers or ())
     headers["accept-encoding"] = "identity"
-
     response: HTTPResponse = urlopen(url, **request_kwargs)
     content_length = get_length(response)
     if content_length == 0 and is_chunked(response):
         content_length = None
-
     fdst: SupportsWrite[bytes]
     if hasattr(file, "write"):
         file = fdst = cast(SupportsWrite[bytes], file)
@@ -328,7 +358,6 @@ def download(
         except FileNotFoundError:
             makedirs(dirname(file), exist_ok=True)
             fdst = open(file, "ab" if resume else "wb")
-
     filesize = 0
     if resume:
         try:
@@ -347,7 +376,6 @@ def download(
                     errno.EIO, 
                     f"file {file!r} is larger than url {url!r}: {filesize} > {content_length} (in bytes)", 
                 )
-
     reporthook_close: None | Callable = None
     if callable(make_reporthook):
         reporthook = make_reporthook(content_length)
@@ -360,7 +388,6 @@ def download(
         reporthook = cast(Callable[[int], Any], reporthook)
     else:
         reporthook = None
-
     try:
         if filesize:
             if is_range_request(response):
@@ -384,6 +411,5 @@ def download(
         response.close()
         if callable(reporthook_close):
             reporthook_close()
-
     return file