python-urlopen 0.0.1__py3-none-any.whl

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 ChenyangGao <https://github.com/ChenyangGao>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python_urlopen-0.0.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 ChenyangGao <https://github.com/ChenyangGao>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python_urlopen-0.0.1.dist-info/METADATA

@@ -0,0 +1,45 @@
+Metadata-Version: 2.1
+Name: python-urlopen
+Version: 0.0.1
+Summary: Python urlopen wrapper.
+Home-page: https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-urlopen
+License: MIT
+Keywords: urlopen
+Author: ChenyangGao
+Author-email: wosiwujm@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: http_response
+Requires-Dist: python-filewrap
+Project-URL: Repository, https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-urlopen
+Description-Content-Type: text/markdown
+
+# Python urlopen wrapper.
+
+## Installation
+
+You can install via [pypi](https://pypi.org/project/python-urlopen/)
+
+```console
+pip install -U python-urlopen
+```
+
+## Usage
+
+```python
+from urlopen import urlopen
+```
+
+

python_urlopen-0.0.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+LICENSE,sha256=o5242_N2TgDsWwFhPn7yr8YJNF7XsJM5NxUMtcT97bc,1100
+urlopen/__init__.py,sha256=LE2hLt-vs6UTcfmShqEYDh22952XwobE7OfLfMBBCe0,7231
+urlopen/__main__.py,sha256=Wj5QLDW2LccpOwBD16pKPOaSj-9r4njgyRIqrlD4aho,2768
+urlopen/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+python_urlopen-0.0.1.dist-info/LICENSE,sha256=o5242_N2TgDsWwFhPn7yr8YJNF7XsJM5NxUMtcT97bc,1100
+python_urlopen-0.0.1.dist-info/METADATA,sha256=_lOD6lD-C0-qrfTSItH8WoMYprOSZ8bxsKaERDYLjjg,1395
+python_urlopen-0.0.1.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+python_urlopen-0.0.1.dist-info/RECORD,,

python_urlopen-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.8.1
+Root-Is-Purelib: true
+Tag: py3-none-any

urlopen/__init__.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+__author__ = "ChenyangGao <https://chenyanggao.github.io>"
+__version__ = (0, 0, 1)
+__all__ = ["urlopen", "download"]
+
+import errno
+
+from collections.abc import Callable, Generator, Mapping, Sequence
+from copy import copy
+from http.client import HTTPResponse
+from inspect import isgenerator
+from json import dumps
+from os import fsdecode, fstat, makedirs, PathLike
+from os.path import abspath, dirname, isdir, join as joinpath
+from shutil import COPY_BUFSIZE # type: ignore
+from ssl import SSLContext, _create_unverified_context
+from typing import cast, Any, Optional
+from urllib.parse import urlencode, urlsplit
+from urllib.request import build_opener, HTTPSHandler, OpenerDirector, Request
+
+from filewrap import bio_skip_bytes, SupportsRead, SupportsWrite
+from http_response import get_filename, get_length, is_chunked, is_range_request
+
+
+if "__del__" not in HTTPResponse.__dict__:
+    setattr(HTTPResponse, "__del__", HTTPResponse.close)
+
+
+def urlopen(
+    url: str | Request, 
+    params: Optional[str | Mapping | Sequence[tuple[Any, Any]]] = None, 
+    data: Optional[bytes | str | Mapping | Sequence[tuple[Any, Any]]] = None, 
+    json: Any = None, 
+    headers: dict[str, str] = {"User-agent": ""}, 
+    method: str = "GET", 
+    timeout: Optional[int | float] = None, 
+    proxy: Optional[tuple[str, str]] = None, 
+    opener: OpenerDirector = build_opener(HTTPSHandler(context=_create_unverified_context())), 
+    context: Optional[SSLContext] = None, 
+    origin: Optional[str] = None, 
+) -> HTTPResponse:
+    if isinstance(url, str) and not urlsplit(url).scheme:
+        if origin:
+            if not url.startswith("/"):
+                url = "/" + url
+            url = origin + url
+    if params:
+        if not isinstance(params, str):
+            params = urlencode(params)
+    params = cast(Optional[str], params)
+    if json is not None:
+        if isinstance(json, bytes):
+            data = json
+        else:
+            data = dumps(json).encode("utf-8")
+        headers = {**headers, "Content-type": "application/json"}
+    elif data is not None:
+        if isinstance(data, bytes):
+            pass
+        elif isinstance(data, str):
+            data = data.encode("utf-8")
+        else:
+            data = urlencode(data).encode("latin-1")
+    data = cast(Optional[bytes], data)
+    if isinstance(url, Request):
+        req = url
+        if params:
+            req.full_url += "?&"["?" in req.full_url] + params
+        if headers:
+            for key, val in headers.items():
+                req.add_header(key, val)
+        if data is not None:
+            req.data = data
+        req.method = method.upper()
+    else:
+        if params:
+            url += "?&"["?" in url] + params
+        req = Request(url, data, headers=headers, method=method.upper())
+    if proxy:
+        req.set_proxy(*proxy)
+    if context:
+        opener = build_opener(HTTPSHandler(context=context))
+    if timeout is None:
+        return opener.open(req)
+    else:
+        return opener.open(req, timeout=timeout)
+
+
+def download(
+    url: str, 
+    file: bytes | str | PathLike | SupportsWrite[bytes] = "", 
+    resume: bool = False, 
+    chunksize: int = COPY_BUFSIZE, 
+    headers: Optional[dict[str, str]] = None, 
+    make_reporthook: Optional[Callable[[Optional[int]], Callable[[int], Any] | Generator[int, Any, Any]]] = None, 
+    **urlopen_kwargs, 
+) -> str | SupportsWrite[bytes]:
+    """Download a URL into a file.
+
+    Example::
+
+        1. use `make_reporthook` to show progress:
+
+            You can use the following function to show progress for the download task
+
+            .. code: python
+
+                from time import perf_counter
+
+                def progress(total=None):
+                    read_num = 0
+                    start_t = perf_counter()
+                    while True:
+                        read_num += yield
+                        speed = read_num / 1024 / 1024 / (perf_counter() - start_t)
+                        print(f"\r\x1b[K{read_num} / {total} | {speed:.2f} MB/s", end="", flush=True)
+
+            Or use the following function for more real-time speed
+
+            .. code: python
+
+                from collections import deque
+                from time import perf_counter
+    
+                def progress(total=None):
+                    dq = deque(maxlen=64)
+                    read_num = 0
+                    dq.append((read_num, perf_counter()))
+                    while True:
+                        read_num += yield
+                        cur_t = perf_counter()
+                        speed = (read_num - dq[0][0]) / 1024 / 1024 / (cur_t - dq[0][1])
+                        print(f"\r\x1b[K{read_num} / {total} | {speed:.2f} MB/s", end="", flush=True)
+                        dq.append((read_num, cur_t))
+    """
+    if headers:
+        headers = {**headers, "Accept-encoding": "identity"}
+    else:
+        headers = {"Accept-encoding": "identity"}
+
+    if chunksize <= 0:
+        chunksize = COPY_BUFSIZE
+
+    resp: HTTPResponse = urlopen(url, headers=headers, **urlopen_kwargs)
+    length = get_length(resp)
+    if length == 0 and is_chunked(resp):
+        length = None
+
+    fdst: SupportsWrite[bytes]
+    if hasattr(file, "write"):
+        file = fdst = cast(SupportsWrite[bytes], file)
+    else:
+        file = abspath(fsdecode(file))
+        if isdir(file):
+            file = joinpath(file, get_filename(resp, "download"))
+        try:
+            fdst = open(file, "ab" if resume else "wb")
+        except FileNotFoundError:
+            makedirs(dirname(file), exist_ok=True)
+            fdst = open(file, "ab" if resume else "wb")
+
+    filesize = 0
+    if resume:
+        try:
+            filesize = fstat(fdst.fileno()).st_size # type: ignore
+        except (AttributeError, OSError):
+            pass
+        else:
+            if filesize == length:
+                return file
+            if filesize and is_range_request(resp):
+                if filesize == length:
+                    return file
+            elif length is not None and filesize > length:
+                raise OSError(errno.EIO, f"file {file!r} is larger than url {url!r}: {filesize} > {length} (in bytes)")
+
+    if make_reporthook:
+        reporthook = make_reporthook(length)
+        if isgenerator(reporthook):
+            next(reporthook)
+            reporthook = reporthook.send
+        reporthook = cast(Callable[[int], Any], reporthook)
+    else:
+        reporthook = None
+
+    try:
+        if filesize:
+            if is_range_request(resp):
+                resp.close()
+                resp = urlopen(url, headers={**headers, "Range": "bytes=%d-" % filesize}, **urlopen_kwargs)
+                if not is_range_request(resp):
+                    raise OSError(errno.EIO, f"range request failed: {url!r}")
+                if reporthook:
+                    reporthook(filesize)
+            elif resume:
+                bio_skip_bytes(resp, filesize, callback=reporthook)
+
+        fsrc_read = resp.read 
+        fdst_write = fdst.write
+        while (chunk := fsrc_read(chunksize)):
+            fdst_write(chunk)
+            if reporthook:
+                reporthook(len(chunk))
+    finally:
+        resp.close()
+
+    return file
+

urlopen/__main__.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+__author__ = "ChenyangGao <https://chenyanggao.github.io>"
+__doc__ = "python url downloader"
+
+from argparse import ArgumentParser, RawTextHelpFormatter
+
+parser = ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
+parser.add_argument("urls", nargs="*", metavar="url", help="URL(s) to be downloaded (one URL per line), if omitted, read from stdin")
+parser.add_argument("-d", "--savedir", default="", help="directory to the downloading files")
+parser.add_argument("-r", "--resume", action="store_true", help="skip downloaded data")
+parser.add_argument("-hs", "--headers", help="dictionary of HTTP Headers to send with")
+parser.add_argument("-v", "--version", action="store_true", help="print the current version")
+args = parser.parse_args()
+if args.version:
+    from urlopen import __version__
+    print(".".join(map(str, __version__)))
+    raise SystemExit(0)
+
+from collections import deque
+from os import makedirs
+from time import perf_counter
+
+from urlopen import download
+
+def headers_str_to_dict(headers: str, /) -> dict[str, str]:
+    return dict(
+        header.split(": ", 1) 
+        for header in headers.strip("\n").split("\n")
+    )
+
+def progress(total=None):
+    dq: deque[tuple[int, float]] = deque(maxlen=64)
+    read_num = 0
+    dq.append((read_num, perf_counter()))
+    while True:
+        read_num += yield
+        cur_t = perf_counter()
+        speed = (read_num - dq[0][0]) / 1024 / 1024 / (cur_t - dq[0][1])
+        if total:
+            percentage = read_num / total * 100
+            print(f"\r\x1b[K{read_num} / {total} | {speed:.2f} MB/s | {percentage:.2f} %", end="", flush=True)
+        else:
+            print(f"\r\x1b[K{read_num} | {speed:.2f} MB/s", end="", flush=True)
+        dq.append((read_num, cur_t))
+
+urls = args.urls
+if not urls:
+    from sys import stdin
+    urls = (l.removesuffix("\n") for l in stdin)
+savedir = args.savedir
+if savedir:
+    makedirs(savedir, exist_ok=True)
+
+try:
+    headers = args.headers
+    if headers is not None:
+        headers = headers_str_to_dict(headers)
+    for url in urls:
+        if not url:
+            continue
+        try:
+            file = download(
+                url, 
+                savedir, 
+                resume=args.resume, 
+                make_reporthook=progress, 
+                headers=headers, 
+            )
+            print(f"\r\x1b[K\x1b[1;32mDOWNLOADED\x1b[0m \x1b[4;34m{url!r}\x1b[0m\n |_ ⏬ \x1b[4;34m{file!r}\x1b[0m")
+        except BaseException as e:
+            print(f"\r\x1b[K\x1b[1;31mERROR\x1b[0m \x1b[4;34m{url!r}\x1b[0m\n  |_ 🙅 \x1b[1;31m{type(e).__qualname__}\x1b[0m: {e}")
+except (EOFError, KeyboardInterrupt):
+    pass
+except BrokenPipeError:
+    from sys import stderr
+    stderr.close()
+