python-urlopen 0.0.8__tar.gz → 0.1.0__tar.gz

{python_urlopen-0.0.8 → python_urlopen-0.1.0}/PKG-INFO

@@ -1,32 +1,36 @@
 Metadata-Version: 2.1
 Name: python-urlopen
-Version: 0.0.8
+Version: 0.1.0
 Summary: Python urlopen wrapper.
-Home-page: https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-urlopen
+Home-page: https://github.com/ChenyangGao/python-modules/tree/main/python-urlopen
 License: MIT
 Keywords: urlopen
 Author: ChenyangGao
 Author-email: wosiwujm@gmail.com
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.12,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Software Development
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: brotli
-Requires-Dist: http_response
-Requires-Dist: python-argtools
-Requires-Dist: python-filewrap
+Requires-Dist: http_response (>=0.0.5)
+Requires-Dist: python-argtools (>=0.0.2)
+Requires-Dist: python-dicttools (>=0.0.1)
+Requires-Dist: python-ensure (>=0.0.1)
+Requires-Dist: python-filewrap (>=0.2.8)
+Requires-Dist: python-http_request (>=0.0.9)
+Requires-Dist: python-undefined (>=0.0.3)
+Requires-Dist: yarl
 Requires-Dist: zstandard
-Project-URL: Repository, https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-urlopen
+Project-URL: Repository, https://github.com/ChenyangGao/python-modules/tree/main/python-urlopen
 Description-Content-Type: text/markdown
 
 # Python urlopen wrapper.

{python_urlopen-0.0.8 → python_urlopen-0.1.0}/pyproject.toml

@@ -1,19 +1,19 @@
 [tool.poetry]
 name = "python-urlopen"
-version = "0.0.8"
+version = "0.1.0"
 description = "Python urlopen wrapper."
 authors = ["ChenyangGao <wosiwujm@gmail.com>"]
 license = "MIT"
 readme = "readme.md"
-homepage = "https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-urlopen"
-repository = "https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-urlopen"
+homepage = "https://github.com/ChenyangGao/python-modules/tree/main/python-urlopen"
+repository = "https://github.com/ChenyangGao/python-modules/tree/main/python-urlopen"
 keywords = ["urlopen"]
 classifiers = [
     "License :: OSI Approved :: MIT License",
     "Development Status :: 5 - Production/Stable",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3 :: Only",
     "Operating System :: OS Independent",
     "Intended Audience :: Developers",
@@ -26,11 +26,16 @@ include = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.12"
 brotli = "*"
-http_response = "*"
-python-argtools = "*"
-python-filewrap = "*"
+http_response = ">=0.0.5"
+python-argtools = ">=0.0.2"
+python-dicttools = ">=0.0.1"
+python-ensure = ">=0.0.1"
+python-filewrap = ">=0.2.8"
+python-http_request = ">=0.0.9"
+python-undefined = ">=0.0.3"
+yarl = "*"
 zstandard = "*"
 
 [tool.poetry.scripts]

{python_urlopen-0.0.8 → python_urlopen-0.1.0}/urlopen/__init__.py

@@ -2,49 +2,63 @@
 # coding: utf-8
 
 __author__ = "ChenyangGao <https://chenyanggao.github.io>"
-__version__ = (0, 0, 8)
+__version__ = (0, 1, 0)
 __all__ = ["urlopen", "request", "download"]
 
 import errno
 
-from collections.abc import Callable, Generator, Iterable, Mapping, Sequence
-from copy import copy
+from collections import UserString
+from collections.abc import Buffer, Callable, Generator, Iterable, Mapping, Sequence
 from gzip import decompress as decompress_gzip
 from http.client import HTTPResponse
 from http.cookiejar import CookieJar
 from inspect import isgenerator
-from json import dumps, loads
 from os import fsdecode, fstat, makedirs, PathLike
 from os.path import abspath, dirname, isdir, join as joinpath
-from re import compile as re_compile
 from shutil import COPY_BUFSIZE # type: ignore
 from socket import getdefaulttimeout, setdefaulttimeout
 from ssl import SSLContext, _create_unverified_context
-from string import punctuation
 from types import EllipsisType
-from typing import cast, Any, Literal
+from typing import cast, overload, Any, Literal
 from urllib.error import HTTPError
-from urllib.parse import quote, urlencode, urlsplit
-from urllib.request import build_opener, HTTPCookieProcessor, HTTPSHandler, OpenerDirector, Request
+from urllib.request import (
+    build_opener, BaseHandler, HTTPCookieProcessor, HTTPSHandler, 
+    HTTPRedirectHandler, OpenerDirector, Request, 
+)
 from zlib import compressobj, DEF_MEM_LEVEL, DEFLATED, MAX_WBITS
 
 from argtools import argcount
-from filewrap import bio_skip_iter, SupportsWrite
-from http_response import get_filename, get_length, is_chunked, is_range_request
+from dicttools import iter_items
+from ensure import ensure_buffer
+from filewrap import bio_skip_iter, bio_chunk_iter, SupportsRead, SupportsWrite
+from http_request import normalize_request_args, SupportsGeturl
+from http_response import (
+    get_filename, get_length, is_chunked, is_range_request, 
+    parse_response, 
+)
+from yarl import URL
+from undefined import undefined, Undefined
 
 
+type string = Buffer | str | UserString
+
 if "__del__" not in HTTPResponse.__dict__:
     setattr(HTTPResponse, "__del__", HTTPResponse.close)
 if "__del__" not in OpenerDirector.__dict__:
     setattr(OpenerDirector, "__del__", OpenerDirector.close)
 
-_opener: None | OpenerDirector = None
-CRE_search_charset = re_compile(r"\bcharset=(?P<charset>[^ ;]+)").search
+_opener: OpenerDirector = build_opener(HTTPSHandler(context=_create_unverified_context()))
+
 
 if getdefaulttimeout() is None:
     setdefaulttimeout(60)
 
 
+class NoRedirectHandler(HTTPRedirectHandler):
+    def redirect_request(self, /, *args, **kwds):
+        return None
+
+
 def decompress_deflate(data: bytes, compresslevel: int = 9) -> bytes:
     # Fork from: https://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations#answer-1089787
     compress = compressobj(
@@ -67,22 +81,9 @@ def decompress_deflate(data: bytes, compresslevel: int = 9) -> bytes:
     return deflated
 
 
-def get_charset(content_type: str, default="utf-8") -> str:
-    match = CRE_search_charset(content_type)
-    if match is None:
-        return "utf-8"
-    return match["charset"]
-
-
-def ensure_ascii_url(url: str, /) -> str:
-    if url.isascii():
-        return url
-    return quote(url, safe=punctuation)
-
-
-def decompress_response(resp: HTTPResponse, /) -> bytes:
-    data = resp.read()
-    content_encoding = resp.headers.get("Content-Encoding")
+def decompress_response(response: HTTPResponse, /) -> bytes:
+    data = response.read()
+    content_encoding = response.headers.get("content-encoding")
     match content_encoding:
         case "gzip":
             data = decompress_gzip(data)
@@ -98,142 +99,174 @@ def decompress_response(resp: HTTPResponse, /) -> bytes:
 
 
 def urlopen(
-    url: str | Request, 
-    method: str = "GET", 
-    params: None | str | Mapping | Sequence[tuple[Any, Any]] = None, 
-    data: None | bytes | str | Mapping | Sequence[tuple[Any, Any]] | Iterable[bytes] = None, 
+    url: string | SupportsGeturl | URL | Request, 
+    method: string = "GET", 
+    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    data: Any = None, 
     json: Any = None, 
-    headers: None | Mapping[str, str] = None, 
-    timeout: None | int | float = None, 
-    cookies: None | CookieJar = None, 
-    proxy: None | tuple[str, str] = None, 
+    headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
+    follow_redirects: bool = True, 
+    proxies: None | Mapping[str, str] | Iterable[tuple[str, str]] = None, 
     context: None | SSLContext = None, 
+    cookies: None | CookieJar = None, 
+    timeout: None | Undefined | float = undefined, 
     opener: None | OpenerDirector = None, 
-    origin: None | str = None, 
+    **_, 
 ) -> HTTPResponse:
-    global _opener
-    if isinstance(url, str) and not urlsplit(url).scheme:
-        if origin:
-            if not url.startswith("/"):
-                url = "/" + url
-            url = origin + url
-    if params:
-        if not isinstance(params, str):
-            params = urlencode(params)
-    params = cast(None | str, params)
-    if json is not None:
-        if isinstance(json, bytes):
-            data = json
-        else:
-            data = dumps(json).encode("utf-8")
-        if headers:
-            headers = {**headers, "Content-type": "application/json"}
-        else:
-            headers = {"Content-type": "application/json"}
-    elif data is not None:
-        if isinstance(data, bytes):
-            pass
-        elif isinstance(data, str):
-            data = data.encode("utf-8")
-        elif isinstance(data, (Mapping, Sequence)):
-            data = urlencode(cast(Mapping | Sequence, data)).encode("latin-1")
-            if headers:
-                headers = {**headers, "Content-type": "application/x-www-form-urlencoded"}
-            else:
-                headers = {"Content-type": "application/x-www-form-urlencoded"}
-    data = cast(None | bytes | Iterable[bytes], data)
     if isinstance(url, Request):
-        req = url
-        if params:
-            req.full_url += "?&"["?" in req.full_url] + params
-        if headers:
-            for key, val in headers.items():
-                req.add_header(key, val)
-        if data is not None:
-            req.data = data
-        req.method = method.upper()
+        request = url
     else:
-        if headers:
-            headers = dict(headers)
-        else:
-            headers = {}
-        if params:
-            url += "?&"["?" in url] + params
-        req = Request(url, data=data, headers=headers, method=method.upper())
-    if proxy:
-        req.set_proxy(*proxy)
+        if isinstance(data, PathLike):
+            data = bio_chunk_iter(open(data, "rb"))
+        elif isinstance(data, SupportsRead):
+            data = map(ensure_buffer, bio_chunk_iter(data))
+        request = Request(**normalize_request_args( # type: ignore
+            method=method, 
+            url=url, 
+            params=params, 
+            data=data, 
+            json=json, 
+            headers=headers, 
+            ensure_ascii=True, 
+        ))
+        if proxies:
+            for host, type in iter_items(proxies):
+                request.set_proxy(host, type)
     if opener is None:
-        if _opener is None:
-            opener = _opener = build_opener(HTTPSHandler(context=_create_unverified_context()))
-        else:
-            opener = _opener
-    if context is not None or cookies is not None:
-        opener = copy(opener)
+        handlers: list[BaseHandler] = []
         if context is not None:
-            opener.add_handler(HTTPSHandler(context=context))
+            handlers.append(HTTPSHandler(context=context))
         if cookies is not None:
-            opener.add_handler(HTTPCookieProcessor(cookies))
-    req.full_url = ensure_ascii_url(req.full_url)
-    if timeout is None:
-        return opener.open(req)
+            handlers.append(HTTPCookieProcessor(cookies))
+        if not follow_redirects:
+            handlers.append(NoRedirectHandler())
+        if handlers:
+            if not isinstance(handlers[0], HTTPSHandler):
+                handlers.insert(0, HTTPSHandler(context=_create_unverified_context()))
+            opener = build_opener(*handlers)
+        else:
+            opener = _opener
+    if timeout is undefined:
+        return opener.open(request)
     else:
-        return opener.open(req, timeout=timeout)
+        return opener.open(request, timeout=cast(None|float, timeout))
 
 
+@overload
 def request(
-    url: str | Request, 
-    method: str = "GET", 
-    parse: None | EllipsisType | bool | Callable = None, 
+    url: string | SupportsGeturl | URL | Request, 
+    method: string = "GET", 
+    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    data: Any = None, 
+    json: Any = None, 
+    headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
+    follow_redirects: bool = True, 
+    raise_for_status: bool = True, 
+    *, 
+    parse: None | EllipsisType = None, 
+    **request_kwargs, 
+) -> HTTPResponse:
+    ...
+@overload
+def request(
+    url: string | SupportsGeturl | URL | Request, 
+    method: string = "GET", 
+    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    data: Any = None, 
+    json: Any = None, 
+    headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
+    follow_redirects: bool = True, 
+    raise_for_status: bool = True, 
+    *, 
+    parse: Literal[False], 
+    **request_kwargs, 
+) -> bytes:
+    ...
+@overload
+def request(
+    url: string | SupportsGeturl | URL | Request, 
+    method: string = "GET", 
+    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    data: Any = None, 
+    json: Any = None, 
+    headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
+    follow_redirects: bool = True, 
     raise_for_status: bool = True, 
-    timeout: None | float = 60, 
+    *, 
+    parse: Literal[True], 
     **request_kwargs, 
-):
-    request_kwargs.pop("stream", None)
+) -> bytes | str | dict | list | int | float | bool | None:
+    ...
+@overload
+def request[T](
+    url: string | SupportsGeturl | URL | Request, 
+    method: string = "GET", 
+    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    data: Any = None, 
+    json: Any = None, 
+    headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
+    follow_redirects: bool = True, 
+    raise_for_status: bool = True, 
+    *, 
+    parse: Callable[[HTTPResponse, bytes], T] | Callable[[HTTPResponse], T], 
+    **request_kwargs, 
+) -> T:
+    ...
+def request[T](
+    url: string | SupportsGeturl | URL | Request, 
+    method: string = "GET", 
+    params: None | string | Mapping | Sequence[tuple[Any, Any]] = None, 
+    data: Any = None, 
+    json: Any = None, 
+    headers: None | Mapping[string, string] | Iterable[tuple[string, string]] = None, 
+    follow_redirects: bool = True, 
+    raise_for_status: bool = True, 
+    *, 
+    parse: None | EllipsisType| bool | Callable[[HTTPResponse, bytes], T] | Callable[[HTTPResponse], T] = None, 
+    **request_kwargs, 
+) -> HTTPResponse | bytes | str | dict | list | int | float | bool | None | T:
     try:
-        resp = urlopen(
+        response = urlopen(
             url=url, 
             method=method, 
-            timeout=timeout, 
+            params=params, 
+            data=data, 
+            json=json, 
+            headers=headers, 
+            follow_redirects=follow_redirects, 
             **request_kwargs, 
         )
     except HTTPError as e:
         if raise_for_status:
             raise
-        resp = getattr(e, "file")
+        response = getattr(e, "file")
     if parse is None:
-        return resp
+        return response
     elif parse is ...:
-        resp.close()
-        return resp
-    with resp:
+        response.close()
+        return response
+    with response:
         if isinstance(parse, bool):
-            data = decompress_response(resp)
+            data = decompress_response(response)
             if parse:
-                content_type = resp.headers.get("Content-Type", "")
-                if content_type == "application/json":
-                    return loads(data)
-                elif content_type.startswith("application/json;"):
-                    return loads(data.decode(get_charset(content_type)))
-                elif content_type.startswith("text/"):
-                    return data.decode(get_charset(content_type))
+                return parse_response(response, data)
             return data
+        ac = argcount(parse)
+        if ac == 1:
+            return cast(Callable[[HTTPResponse], T], parse)(response)
         else:
-            ac = argcount(parse)
-            with resp:
-                if ac == 1:
-                    return parse(resp)
-                else:
-                    return parse(resp, decompress_response(resp))
+            return cast(Callable[[HTTPResponse, bytes], T], parse)(
+                response, decompress_response(response))
 
 
 def download(
-    url: str, 
+    url: string | SupportsGeturl | URL | Request, 
     file: bytes | str | PathLike | SupportsWrite[bytes] = "", 
     resume: bool = False, 
     chunksize: int = COPY_BUFSIZE, 
-    headers: None | Mapping[str, str] = None, 
+    headers: None | Mapping[str, str] | Iterable[tuple[str, str]] = None, 
     make_reporthook: None | Callable[[None | int], Callable[[int], Any] | Generator[int, Any, Any]] = None, 
-    **urlopen_kwargs, 
+    **request_kwargs, 
 ) -> str | SupportsWrite[bytes]:
     """Download a URL into a file.
 
@@ -273,17 +306,14 @@ def download(
                         print(f"\r\x1b[K{read_num} / {total} | {speed:.2f} MB/s", end="", flush=True)
                         dq.append((read_num, cur_t))
     """
-    if headers:
-        headers = {**headers, "Accept-encoding": "identity"}
-    else:
-        headers = {"Accept-encoding": "identity"}
-
     if chunksize <= 0:
         chunksize = COPY_BUFSIZE
+    headers = request_kwargs["headers"] = dict(headers or ())
+    headers["accept-encoding"] = "identity"
 
-    resp: HTTPResponse = urlopen(url, headers=headers, **urlopen_kwargs)
-    content_length = get_length(resp)
-    if content_length == 0 and is_chunked(resp):
+    response: HTTPResponse = urlopen(url, **request_kwargs)
+    content_length = get_length(response)
+    if content_length == 0 and is_chunked(response):
         content_length = None
 
     fdst: SupportsWrite[bytes]
@@ -292,7 +322,7 @@ def download(
     else:
         file = abspath(fsdecode(file))
         if isdir(file):
-            file = joinpath(file, get_filename(resp, "download"))
+            file = joinpath(file, get_filename(response, "download"))
         try:
             fdst = open(file, "ab" if resume else "wb")
         except FileNotFoundError:
@@ -309,7 +339,7 @@ def download(
         else:
             if filesize == content_length:
                 return file
-            if filesize and is_range_request(resp):
+            if filesize and is_range_request(response):
                 if filesize == content_length:
                     return file
             elif content_length is not None and filesize > content_length:
@@ -333,25 +363,25 @@ def download(
 
     try:
         if filesize:
-            if is_range_request(resp):
-                resp.close()
-                resp = urlopen(url, headers={**headers, "Range": "bytes=%d-" % filesize}, **urlopen_kwargs)
-                if not is_range_request(resp):
+            if is_range_request(response):
+                response.close()
+                response = urlopen(url, headers={**headers, "Range": "bytes=%d-" % filesize}, **request_kwargs)
+                if not is_range_request(response):
                     raise OSError(errno.EIO, f"range request failed: {url!r}")
                 if reporthook is not None:
                     reporthook(filesize)
             elif resume:
-                for _ in bio_skip_iter(resp, filesize, callback=reporthook):
+                for _ in bio_skip_iter(response, filesize, callback=reporthook):
                     pass
 
-        fsrc_read = resp.read 
+        fsrc_read = response.read 
         fdst_write = fdst.write
         while (chunk := fsrc_read(chunksize)):
             fdst_write(chunk)
             if reporthook is not None:
                 reporthook(len(chunk))
     finally:
-        resp.close()
+        response.close()
         if callable(reporthook_close):
             reporthook_close()