python-download 0.0.1__tar.gz

python_download-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 ChenyangGao <https://github.com/ChenyangGao>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python_download-0.0.1/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.1
+Name: python-download
+Version: 0.0.1
+Summary: Python functions for download.
+Home-page: https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-download
+License: MIT
+Keywords: python-download,posixpath
+Author: ChenyangGao
+Author-email: wosiwujm@gmail.com
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: aiohttp_client_request
+Requires-Dist: http_response
+Requires-Dist: python-asynctools
+Requires-Dist: python-concurrenttools
+Requires-Dist: python-filewrap
+Requires-Dist: python-http_request
+Requires-Dist: python-iterutils
+Requires-Dist: python-urlopen
+Requires-Dist: requests_request
+Project-URL: Repository, https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-download
+Description-Content-Type: text/markdown
+
+# Python functions for download..
+
+## Installation
+
+You can install from [pypi](https://pypi.org/project/python-download/)
+
+```console
+pip install -U python-download
+```
+
+## Usage
+
+```python
+import download
+```
+

python_download-0.0.1/download/__init__.py

@@ -0,0 +1,444 @@
+#!/usr/bin/env python3
+# encoding: utf
+
+__author__ = "ChenyangGao <https://chenyanggao.github.io>"
+__version__ = (0, 0, 1)
+__all__ = [
+    "DownloadTask", "download_iter", "download", "requests_download", 
+    "download_async_iter", "async_download", 
+]
+
+# NOTE https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
+# NOTE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges
+# NOTE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range
+# NOTE https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/206
+# NOTE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+# NOTE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range
+# NOTE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Range
+# NOTE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding
+# NOTE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
+
+import errno
+
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Iterator
+from inspect import isasyncgen, isgenerator
+from os import fsdecode, fstat, makedirs, PathLike
+from os.path import abspath, dirname, isdir, join as joinpath
+from shutil import COPY_BUFSIZE # type: ignore
+from threading import Event
+from typing import cast, Any, NamedTuple, Optional, Self
+
+from aiohttp_client_request import request as aiohttp_request
+from asynctools import ensure_async, as_thread
+from concurrenttools import run_as_thread
+from filewrap import bio_skip_iter, bio_skip_async_iter, SupportsRead, SupportsWrite
+from http_request import headers_str_to_dict
+from http_response import get_filename, get_length, is_chunked, is_range_request
+from iterutils import cut_iter
+from requests_request import request as requests_request
+from urlopen import urlopen
+
+try:
+    from aiofile import async_open, FileIOWrapperBase
+    aiofile_installed = True
+except ImportError:
+    aiofile_installed = False
+else:
+    if "__getattr__" not in FileIOWrapperBase.__dict__:
+        setattr(FileIOWrapperBase, "__getattr__", lambda self, attr, /: getattr(self.file, attr))
+
+
+class DownloadProgress(NamedTuple):
+    total: int
+    downloaded: int
+    skipped: int
+    last_incr: int = 0
+    extra: Any = None
+
+    @property
+    def completed(self, /) -> int:
+        return self.downloaded + self.skipped
+
+    @property
+    def remaining(self, /) -> int:
+        return max(0, self.total - self.completed)
+
+    @property
+    def ratio(self, /) -> float:
+        return self.completed / self.total
+
+    @property
+    def task_done(self, /) -> bool:
+        return self.completed >= self.total
+
+
+class DownloadTask:
+
+    def __init__(self, /, gen, submit=run_as_thread):
+        if not callable(submit):
+            submit = submit.submit
+        self._submit = submit
+        self._state = "PENDING"
+        self._gen = gen
+        self._done_event = Event()
+
+    def __repr__(self, /) -> str:
+        return f"<{type(self).__qualname__} :: state={self.state!r} progress={self.progress!r}>"
+
+    @classmethod
+    def create_task(
+        cls, 
+        /, 
+        *args, 
+        submit=run_as_thread, 
+        **kwargs, 
+    ) -> Self:
+        return cls(download_iter(*args, **kwargs), submit=submit)
+
+    @property
+    def closed(self, /) -> bool:
+        return self._state in ("CANCELED", "FAILED", "FINISHED")
+
+    @property
+    def progress(self, /) -> Optional[DownloadProgress]:
+        return self.__dict__.get("_progress")
+
+    @property
+    def result(self, /):
+        self._done_event.wait()
+        return self._result
+
+    @result.setter
+    def result(self, val, /):
+        self._result = val
+        self._done_event.set()
+
+    @property
+    def state(self, /) -> str:
+        return self._state
+
+    def close(self, /):
+        if self._state in ("CANCELED", "FAILED", "FINISHED"):
+            pass
+        else:
+            state = self._state
+            self._state = "CANCELED"
+            if state != "RUNNING":
+                self.run()
+
+    def pause(self, /):
+        if self._state in ("PAUSED", "RUNNING"):
+            self._state = "PAUSED"
+        else:
+            raise RuntimeError(f"can't pause when state={self._state!r}")
+
+    def _run(self, /):
+        if self._state in ("PENDING", "PAUSED"):
+            self._state = "RUNNING"
+        else:
+            raise RuntimeError(f"can't run when state={self._state!r}")
+        gen = self._gen
+        try:
+            while self._state == "RUNNING":
+                self._progress = next(gen)
+        except KeyboardInterrupt:
+            raise
+        except StopIteration as exc:
+            self._state = "FINISHED"
+            self.result = exc.value
+        except BaseException as exc:
+            self._state = "FAILED"
+            self.result = exc
+        else:
+            if self._state == "CANCELED":
+                try:
+                    gen.close()
+                finally:
+                    self.result = None
+
+    def run(self, /):
+        return self._submit(self._run)
+
+    def run_wait(self, /):
+        if not self._done_event.is_set():
+            if self._state == "RUNNING":
+                self._done_event.wait()
+            else:
+                self._run()
+
+
+def download_iter(
+    url: str | Callable[[], str], 
+    file: bytes | str | PathLike | SupportsWrite[bytes] = "", 
+    resume: bool = False, 
+    chunksize: int = COPY_BUFSIZE, 
+    headers: None | dict[str, str] | Callable[[], dict[str, str]] = None, 
+    urlopen: Callable = urlopen, 
+) -> Generator[DownloadProgress, None, None]:
+    """
+    """
+    if not isinstance(url, str):
+        url = url()
+
+    if callable(headers):
+        headers = headers()
+    if headers:
+        headers = {**headers, "Accept-Encoding": "identity"}
+    else:
+        headers = {"Accept-Encoding": "identity"}
+
+    if chunksize <= 0:
+        chunksize = COPY_BUFSIZE
+
+    resp = urlopen(url, headers=headers)
+    try:
+        length = get_length(resp)
+        if length == 0 and is_chunked(resp):
+            length = None
+
+        fdst: SupportsWrite[bytes]
+        if hasattr(file, "write"):
+            file = fdst = cast(SupportsWrite[bytes], file)
+        else:
+            file = abspath(fsdecode(file))
+            if isdir(file):
+                file = joinpath(file, get_filename(resp, "download"))
+            try:
+                fdst = open(file, "ab" if resume else "wb")
+            except FileNotFoundError:
+                makedirs(dirname(file), exist_ok=True)
+                fdst = open(file, "ab" if resume else "wb")
+
+        extra = {"url": url, "file": file, "resume": resume}
+
+        filesize = 0
+        if resume:
+            try:
+                filesize = fstat(fdst.fileno()).st_size # type: ignore
+            except (AttributeError, OSError):
+                pass
+            else:
+                if filesize == length:
+                    yield DownloadProgress(length, 0, length, length, extra)
+                    return
+                elif length is not None and filesize > length:
+                    raise OSError(errno.EIO, f"file {file!r} is larger than url {url!r}: {filesize} > {length} (in bytes)")
+        elif length == 0:
+            yield DownloadProgress(0, 0, 0, 0, extra)
+            return
+
+        if filesize and is_range_request(resp):
+            resp.close()
+            resp = urlopen(url, headers={**headers, "Range": "bytes=%d-" % filesize})
+            if not is_range_request(resp):
+                raise OSError(errno.EIO, f"range request failed: {url!r}")
+
+        yield DownloadProgress(length or 0, 0, 0, 0, extra)
+
+        length_downloaded = 0
+        length_skipped = 0
+
+        fsrc_read = resp.read
+        fdst_write = fdst.write
+        if filesize:
+            if is_range_request(resp):
+                length_skipped = filesize
+                yield DownloadProgress(length or length_skipped, 0, length_skipped, length_skipped, extra)
+            else:
+                for skiplen in bio_skip_iter(resp, filesize):
+                    length_skipped += skiplen
+                    yield DownloadProgress(length or length_skipped, 0, length_skipped, skiplen, extra)
+
+        while (chunk := fsrc_read(chunksize)):
+            fdst_write(chunk)
+            downlen = len(chunk)
+            length_downloaded += downlen
+            yield DownloadProgress(length or (length_skipped + length_downloaded), length_downloaded, length_skipped, downlen, extra)
+    finally:
+        resp.close()
+
+
+def download(
+    url: str | Callable[[], str], 
+    file: bytes | str | PathLike | SupportsWrite[bytes] = "", 
+    resume: bool = False, 
+    chunksize: int = COPY_BUFSIZE, 
+    headers: None | dict[str, str] | Callable[[], dict[str, str]] = None, 
+    urlopen: Callable = urlopen, 
+    make_reporthook: None | Callable[[None | int], Callable[[int], Any] | Generator[int, Any, Any]] = None, 
+):
+    """
+    """
+    gen = download_iter(url, file, resume=resume, chunksize=chunksize, headers=headers, urlopen=urlopen)
+    if make_reporthook:
+        progress = next(gen)
+        reporthook = make_reporthook(progress.total)
+        if isgenerator(reporthook):
+            next(reporthook)
+            reporthook = reporthook.send
+        reporthook = cast(Callable[[int], Any], reporthook)
+        reporthook(progress.last_incr)
+    else:
+        reporthook = None
+
+    for progress in gen:
+        reporthook and reporthook(progress.last_incr)
+
+
+def requests_download(
+    url: str | Callable[[], str], 
+    urlopen: Callable = requests_request, 
+    **kwargs, 
+):
+    """
+    """
+    def urlopen_wrapper(url, headers):
+        resp = urlopen(url, headers=headers, stream=True)
+        resp.raise_for_status()
+        resp.read = resp.raw.read
+        return resp
+    return download(url, urlopen=urlopen_wrapper, **kwargs)
+
+
+async def download_async_iter(
+    url: str | Callable[[], str], 
+    file: bytes | str | PathLike | SupportsWrite[bytes] = "", 
+    resume: bool = False, 
+    chunksize: int = COPY_BUFSIZE, 
+    headers: None | dict[str, str] | Callable[[], dict[str, str]] = None, 
+    urlopen: Callable = aiohttp_request, 
+) -> AsyncGenerator[DownloadProgress, None]:
+    """
+    """
+    if not isinstance(url, str):
+        url = await ensure_async(url)()
+
+    if callable(headers):
+        headers = await ensure_async(headers)()
+    if headers:
+        headers = {**headers, "Accept-Encoding": "identity"}
+    else:
+        headers = {"Accept-Encoding": "identity"}
+
+    if chunksize <= 0:
+        chunksize = COPY_BUFSIZE
+
+    urlopen = ensure_async(urlopen)
+
+    resp = await urlopen(url, headers=headers)
+    file_async_close: None | Callable = None
+    try:
+        length = get_length(resp)
+        if length == 0 and is_chunked(resp):
+            length = None
+
+        fdst: SupportsWrite[bytes]
+        if hasattr(file, "write"):
+            file = fdst = cast(SupportsWrite[bytes], file)
+        else:
+            file = abspath(fsdecode(file))
+            if isdir(file):
+                file = joinpath(file, get_filename(resp, "download"))
+            if aiofile_installed:
+                try:
+                    fdst = await async_open(file, "ab" if resume else "wb")
+                except FileNotFoundError:
+                    makedirs(dirname(file), exist_ok=True)
+                    fdst = await async_open(file, "ab" if resume else "wb")
+                file_async_close = fdst.close
+            else:
+                try:
+                    fdst = open(file, "ab" if resume else "wb")
+                except FileNotFoundError:
+                    makedirs(dirname(file), exist_ok=True)
+                    fdst = open(file, "ab" if resume else "wb")
+                file_async_close = as_thread(fdst.close)
+
+        extra = {"url": url, "file": file, "resume": resume}
+        filesize = 0
+        if resume:
+            try:
+                filesize = fstat(fdst.fileno()).st_size # type: ignore
+            except (AttributeError, OSError):
+                pass
+            else:
+                if filesize == length:
+                    yield DownloadProgress(length, 0, length, length, extra)
+                    return
+                elif length is not None and filesize > length:
+                    raise OSError(errno.EIO, f"file {file!r} is larger than url {url!r}: {filesize} > {length} (in bytes)")
+        elif length == 0:
+            yield DownloadProgress(0, 0, 0, 0, extra)
+            return
+
+        if filesize and is_range_request(resp):
+            await ensure_async(resp.close)()
+            resp = await urlopen(url, headers={**headers, "Range": "bytes=%d-" % filesize})
+            if not is_range_request(resp):
+                raise OSError(errno.EIO, f"range request failed: {url!r}")
+
+        yield DownloadProgress(length or 0, 0, 0, 0, extra)
+
+        length_downloaded = 0
+        length_skipped = 0
+
+        fsrc_read = ensure_async(resp.read)
+        fdst_write = ensure_async(fdst.write)
+        if filesize:
+            if is_range_request(resp):
+                length_skipped = filesize
+                yield DownloadProgress(length or length_skipped, 0, length_skipped, length_skipped, extra)
+            else:
+                async for skiplen in bio_skip_async_iter(resp, filesize):
+                    length_skipped += skiplen
+                    yield DownloadProgress(length or length_skipped, 0, length_skipped, skiplen, extra)
+
+        while (chunk := (await fsrc_read(chunksize))):
+            await fdst_write(chunk)
+            downlen = len(chunk)
+            length_downloaded += downlen
+            yield DownloadProgress(length or (length_skipped + length_downloaded), length_downloaded, length_skipped, downlen, extra)
+    finally:
+        await ensure_async(resp.close)()
+        if file_async_close:
+            await file_async_close()
+
+
+async def async_download(
+    url: str | Callable[[], str], 
+    file: bytes | str | PathLike | SupportsWrite[bytes] = "", 
+    resume: bool = False, 
+    chunksize: int = COPY_BUFSIZE, 
+    headers: None | dict[str, str] | Callable[[], dict[str, str]] = None, 
+    urlopen: Callable = urlopen, 
+    make_reporthook: None | Callable[[None | int], Callable[[int], Any] | Generator[int, Any, Any] | AsyncGenerator[int, Any]] = None, 
+):
+    """
+    """
+    gen = download_async_iter(url, file, resume=resume, chunksize=chunksize, headers=headers, urlopen=urlopen)
+    reporthook_close: None | Callable = None
+    if make_reporthook:
+        progress = await anext(gen)
+        reporthook = make_reporthook(progress.total)
+        if isasyncgen(reporthook):
+            await anext(reporthook)
+            reporthook_close = reporthook.aclose
+            reporthook = reporthook.asend
+        elif isgenerator(reporthook):
+            await as_thread(next)(reporthook)
+            reporthook_close = as_thread(reporthook.close)
+            reporthook = as_thread(reporthook.send)
+        else:
+            reporthook = ensure_async(cast(Callable, reporthook))
+        await reporthook(progress.last_incr)
+    else:
+        reporthook = None
+
+    try:
+        async for progress in gen:
+            reporthook and await reporthook(progress.last_incr)
+    finally:
+        await gen.aclose()
+        if reporthook_close:
+            await reporthook_close()
+

python_download-0.0.1/download/__main__.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+__author__ = "ChenyangGao <https://chenyanggao.github.io>"
+__doc__ = "python url downloader"
+
+from argparse import ArgumentParser, RawTextHelpFormatter
+from collections import deque
+from collections.abc import Callable
+from os import makedirs
+from time import perf_counter
+
+from . import download, requests_download, __version__
+
+
+def parse_args():
+    parser = ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
+    parser.add_argument("urls", metavar="url", nargs="?", help="URL(s) to be downloaded (one URL per line), if omitted, read from stdin")
+    parser.add_argument("-d", "--savedir", default="", help="path to the downloaded file")
+    parser.add_argument("-r", "--resume", action="store_true", help="skip downloaded data")
+    parser.add_argument("-hs", "--headers", help="dictionary of HTTP Headers to send with")
+    parser.add_argument("-rq", "--use-requests", action="store_true", help="use `requests` module")
+    parser.add_argument("-v", "--version", action="store_true", help="print the current version")
+    args = parser.parse_args()
+    if args.version:
+        print(".".join(map(str, __version__)))
+        raise SystemExit(0)
+    return args
+
+
+def headers_str_to_dict(headers: str, /) -> dict[str, str]:
+    return dict(
+        header.split(": ", 1) 
+        for header in headers.strip("\n").split("\n")
+    )
+
+
+def progress(total=None):
+    dq: deque[tuple[int, float]] = deque(maxlen=64)
+    read_num = 0
+    dq.append((read_num, perf_counter()))
+    while True:
+        read_num += yield
+        cur_t = perf_counter()
+        speed = (read_num - dq[0][0]) / 1024 / 1024 / (cur_t - dq[0][1])
+        if total:
+            percentage = read_num / total * 100
+            print(f"\r\x1b[K{read_num} / {total} | {speed:.2f} MB/s | {percentage:.2f} %", end="", flush=True)
+        else:
+            print(f"\r\x1b[K{read_num} | {speed:.2f} MB/s", end="", flush=True)
+        dq.append((read_num, cur_t))
+
+
+def main():
+    args = parse_args()
+
+    if args.urls:
+        urls = args.urls.splitlines()
+    else:
+        from sys import stdin
+        urls = (l.removesuffix("\n") for l in stdin)
+
+    savedir = args.savedir
+    if savedir:
+        makedirs(savedir, exist_ok=True)
+
+    if args.use_requests:
+        downloader: Callable = requests_download
+    else:
+        downloader = download
+
+    try:
+        headers = args.headers
+        if headers is not None:
+            headers = headers_str_to_dict(headers)
+        for url in urls:
+            if not url:
+                continue
+            try:
+                file = downloader(
+                    url, 
+                    file=savedir, 
+                    resume=args.resume, 
+                    make_reporthook=progress, 
+                    headers=headers, 
+                )
+                print(f"\r\x1b[K\x1b[1;32mDOWNLOADED\x1b[0m \x1b[4;34m{url!r}\x1b[0m\n |_ ⏬ \x1b[4;34m{file!r}\x1b[0m")
+            except BaseException as e:
+                print(f"\r\x1b[K\x1b[1;31mERROR\x1b[0m \x1b[4;34m{url!r}\x1b[0m\n  |_ 🙅 \x1b[1;31m{type(e).__qualname__}\x1b[0m: {e}")
+    except (EOFError, KeyboardInterrupt):
+        pass
+    except BrokenPipeError:
+        from sys import stderr
+        stderr.close()
+
+
+if __name__ == "__main__":
+    main()
+

python_download-0.0.1/pyproject.toml

@@ -0,0 +1,49 @@
+[tool.poetry]
+name = "python-download"
+version = "0.0.1"
+description = "Python functions for download."
+authors = ["ChenyangGao <wosiwujm@gmail.com>"]
+license = "MIT"
+readme = "readme.md"
+homepage = "https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-download"
+repository = "https://github.com/ChenyangGao/web-mount-packs/tree/main/python-module/python-download"
+keywords = ["python-download", "posixpath"]
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Development Status :: 5 - Production/Stable",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3 :: Only",
+    "Operating System :: OS Independent",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+include = [
+    "LICENSE",
+]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+aiohttp_client_request = "*"
+http_response = "*"
+python-asynctools = "*"
+python-concurrenttools = "*"
+python-filewrap = "*"
+python-iterutils = "*"
+python-http_request = "*"
+python-urlopen = "*"
+requests_request = "*"
+
+[tool.poetry.scripts]
+python-download = "download.__main__:main"
+download = "download.__main__:main"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[[tool.poetry.packages]]
+include = "download"

python_download-0.0.1/readme.md

@@ -0,0 +1,15 @@
+# Python functions for download..
+
+## Installation
+
+You can install from [pypi](https://pypi.org/project/python-download/)
+
+```console
+pip install -U python-download
+```
+
+## Usage
+
+```python
+import download
+```