Jedutils 0.1.3__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {Jedutils-0.1.3 → Jedutils-0.1.4}/Jedutils.egg-info/PKG-INFO +1 -1
- {Jedutils-0.1.3 → Jedutils-0.1.4}/Jedutils.egg-info/SOURCES.txt +2 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/PKG-INFO +1 -1
- {Jedutils-0.1.3 → Jedutils-0.1.4}/jedutils/__init__.py +1 -1
- Jedutils-0.1.4/jedutils/asyncio/__init__.py +4 -0
- Jedutils-0.1.4/jedutils/asyncio/network/__init__.py +2 -0
- Jedutils-0.1.4/jedutils/asyncio/network/_dl.py +92 -0
- Jedutils-0.1.3/jedutils/asyncio/__init__.py +0 -3
- {Jedutils-0.1.3 → Jedutils-0.1.4}/Jedutils.egg-info/dependency_links.txt +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/Jedutils.egg-info/top_level.txt +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/LICENSE +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/MANIFEST.in +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/README.md +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/jedutils/asyncio/_async_redis_pipe.py +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/jedutils/asyncio/_run_async.py +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/jedutils/strings/__init__.py +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/jedutils/strings/_random_string.py +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/setup.cfg +0 -0
- {Jedutils-0.1.3 → Jedutils-0.1.4}/setup.py +0 -0
|
@@ -10,5 +10,7 @@ jedutils/__init__.py
|
|
|
10
10
|
jedutils/asyncio/__init__.py
|
|
11
11
|
jedutils/asyncio/_async_redis_pipe.py
|
|
12
12
|
jedutils/asyncio/_run_async.py
|
|
13
|
+
jedutils/asyncio/network/__init__.py
|
|
14
|
+
jedutils/asyncio/network/_dl.py
|
|
13
15
|
jedutils/strings/__init__.py
|
|
14
16
|
jedutils/strings/_random_string.py
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import aiohttp
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
async def download_file(
|
|
7
|
+
url: str,
|
|
8
|
+
filename: str,
|
|
9
|
+
chunk_size: int = 8192,
|
|
10
|
+
headers: dict = None,
|
|
11
|
+
ignore_status: bool = False,
|
|
12
|
+
timeout: float = None,
|
|
13
|
+
progress_callback: callable = None,
|
|
14
|
+
thread_pool=None,
|
|
15
|
+
) -> None:
|
|
16
|
+
"""
|
|
17
|
+
Asynchronously download a file from the given URL and saves it to the specified filename using chunked downloading
|
|
18
|
+
|
|
19
|
+
Example:
|
|
20
|
+
Download a large file asynchronously:
|
|
21
|
+
|
|
22
|
+
.. code-block:: python
|
|
23
|
+
|
|
24
|
+
async def my_callback(downloaded, expected_size):
|
|
25
|
+
print(f"Downloaded: {downloaded}/{expected_size}")
|
|
26
|
+
|
|
27
|
+
await download_file('https://example.com/large_file.zip', 'my_large_file.zip', progress_callback=my_callback)
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
url (``str``):
|
|
31
|
+
The URL of the file to download
|
|
32
|
+
|
|
33
|
+
filename (``str``, *optional*):
|
|
34
|
+
The filename to save the downloaded file as
|
|
35
|
+
|
|
36
|
+
chunk_size (``int``, *optional*):
|
|
37
|
+
The size of each chunk to download in bytes. Default is 8192 bytes (8 KB)
|
|
38
|
+
|
|
39
|
+
headers (``dict``, *optional*):
|
|
40
|
+
A dictionary of headers to send with the request. Default is ``None``
|
|
41
|
+
|
|
42
|
+
ignore_status (``bool``, *optional*):
|
|
43
|
+
Whether to ignore errors in the response from the server. Default is ``False``
|
|
44
|
+
|
|
45
|
+
timeout (``float``, *optional*):
|
|
46
|
+
A timeout for the request. Default is ``None``
|
|
47
|
+
|
|
48
|
+
progress_callback (``callable``, *optional*):
|
|
49
|
+
A callable function to be called with the download progress, in bytes
|
|
50
|
+
|
|
51
|
+
thread_pool:
|
|
52
|
+
A thread pool to use for blocking operations. Default is ``None``
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
``None``
|
|
56
|
+
|
|
57
|
+
Raises:
|
|
58
|
+
:py:class:`aiohttp.ClientResponseError`
|
|
59
|
+
"""
|
|
60
|
+
async with aiohttp.ClientSession() as session:
|
|
61
|
+
async with session.get(url, timeout=timeout, headers=headers) as response:
|
|
62
|
+
if not ignore_status:
|
|
63
|
+
response.raise_for_status()
|
|
64
|
+
|
|
65
|
+
expected_size = int(response.headers.get("content-length", 0))
|
|
66
|
+
downloaded_size = 0
|
|
67
|
+
loop = asyncio.get_event_loop()
|
|
68
|
+
|
|
69
|
+
if progress_callback:
|
|
70
|
+
progress_callback(downloaded_size, expected_size)
|
|
71
|
+
|
|
72
|
+
if not filename:
|
|
73
|
+
filename = os.path.basename(url)
|
|
74
|
+
if not filename:
|
|
75
|
+
raise ValueError("Can't find filename")
|
|
76
|
+
|
|
77
|
+
f = None
|
|
78
|
+
try:
|
|
79
|
+
f = await loop.run_in_executor(thread_pool, open, filename, "wb")
|
|
80
|
+
|
|
81
|
+
async for chunk in response.content.iter_chunked(chunk_size):
|
|
82
|
+
await loop.run_in_executor(thread_pool, f.write, chunk)
|
|
83
|
+
|
|
84
|
+
downloaded_size += len(chunk)
|
|
85
|
+
if progress_callback:
|
|
86
|
+
progress_callback(downloaded_size, expected_size)
|
|
87
|
+
|
|
88
|
+
finally:
|
|
89
|
+
if f:
|
|
90
|
+
await loop.run_in_executor(thread_pool, f.close)
|
|
91
|
+
|
|
92
|
+
return None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|