sticker-convert 2.13.3.0__py3-none-any.whl → 2.17.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sticker_convert/__main__.py +24 -27
- sticker_convert/auth/__init__.py +0 -0
- sticker_convert/auth/auth_base.py +19 -0
- sticker_convert/{utils/auth/get_discord_auth.py → auth/auth_discord.py} +149 -118
- sticker_convert/{utils/auth/get_kakao_auth.py → auth/auth_kakao_android_login.py} +331 -330
- sticker_convert/auth/auth_kakao_desktop_login.py +327 -0
- sticker_convert/{utils/auth/get_kakao_desktop_auth.py → auth/auth_kakao_desktop_memdump.py} +281 -263
- sticker_convert/{utils/auth/get_line_auth.py → auth/auth_line.py} +98 -80
- sticker_convert/{utils/auth/get_signal_auth.py → auth/auth_signal.py} +139 -135
- sticker_convert/auth/auth_telethon.py +161 -0
- sticker_convert/{utils/auth/get_viber_auth.py → auth/auth_viber.py} +250 -235
- sticker_convert/{utils/auth → auth}/telegram_api.py +736 -675
- sticker_convert/cli.py +623 -608
- sticker_convert/converter.py +1093 -1084
- sticker_convert/definitions.py +4 -0
- sticker_convert/downloaders/download_band.py +111 -110
- sticker_convert/downloaders/download_base.py +171 -166
- sticker_convert/downloaders/download_discord.py +92 -91
- sticker_convert/downloaders/download_kakao.py +417 -404
- sticker_convert/downloaders/download_line.py +484 -475
- sticker_convert/downloaders/download_ogq.py +80 -79
- sticker_convert/downloaders/download_signal.py +108 -105
- sticker_convert/downloaders/download_telegram.py +56 -55
- sticker_convert/downloaders/download_viber.py +121 -120
- sticker_convert/gui.py +788 -873
- sticker_convert/gui_components/frames/comp_frame.py +180 -166
- sticker_convert/gui_components/frames/config_frame.py +156 -113
- sticker_convert/gui_components/frames/control_frame.py +32 -30
- sticker_convert/gui_components/frames/cred_frame.py +232 -233
- sticker_convert/gui_components/frames/input_frame.py +139 -137
- sticker_convert/gui_components/frames/output_frame.py +112 -110
- sticker_convert/gui_components/frames/right_clicker.py +25 -23
- sticker_convert/gui_components/windows/advanced_compression_window.py +757 -757
- sticker_convert/gui_components/windows/base_window.py +7 -2
- sticker_convert/gui_components/windows/discord_get_auth_window.py +79 -82
- sticker_convert/gui_components/windows/kakao_get_auth_window.py +511 -321
- sticker_convert/gui_components/windows/line_get_auth_window.py +94 -102
- sticker_convert/gui_components/windows/signal_get_auth_window.py +84 -89
- sticker_convert/gui_components/windows/viber_get_auth_window.py +168 -168
- sticker_convert/ios-message-stickers-template/.github/FUNDING.yml +3 -3
- sticker_convert/ios-message-stickers-template/README.md +10 -10
- sticker_convert/ios-message-stickers-template/stickers/Info.plist +43 -43
- sticker_convert/ios-message-stickers-template/stickers StickerPackExtension/Info.plist +31 -31
- sticker_convert/ios-message-stickers-template/stickers StickerPackExtension/Stickers.xcstickers/Contents.json +6 -6
- sticker_convert/ios-message-stickers-template/stickers StickerPackExtension/Stickers.xcstickers/Sticker Pack.stickerpack/Contents.json +20 -20
- sticker_convert/ios-message-stickers-template/stickers StickerPackExtension/Stickers.xcstickers/Sticker Pack.stickerpack/Sticker 1.sticker/Contents.json +9 -9
- sticker_convert/ios-message-stickers-template/stickers StickerPackExtension/Stickers.xcstickers/Sticker Pack.stickerpack/Sticker 2.sticker/Contents.json +9 -9
- sticker_convert/ios-message-stickers-template/stickers StickerPackExtension/Stickers.xcstickers/Sticker Pack.stickerpack/Sticker 3.sticker/Contents.json +9 -9
- sticker_convert/ios-message-stickers-template/stickers StickerPackExtension/Stickers.xcstickers/iMessage App Icon.stickersiconset/Contents.json +91 -91
- sticker_convert/ios-message-stickers-template/stickers.xcodeproj/project.pbxproj +364 -364
- sticker_convert/ios-message-stickers-template/stickers.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 -7
- sticker_convert/ios-message-stickers-template/stickers.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -8
- sticker_convert/ios-message-stickers-template/stickers.xcodeproj/xcuserdata/niklaspeterson.xcuserdatad/xcschemes/xcschememanagement.plist +14 -14
- sticker_convert/job.py +166 -130
- sticker_convert/job_option.py +1 -0
- sticker_convert/locales/en_US/LC_MESSAGES/base.mo +0 -0
- sticker_convert/locales/ja_JP/LC_MESSAGES/base.mo +0 -0
- sticker_convert/locales/zh_CN/LC_MESSAGES/base.mo +0 -0
- sticker_convert/locales/zh_TW/LC_MESSAGES/base.mo +0 -0
- sticker_convert/py.typed +0 -0
- sticker_convert/resources/NotoColorEmoji.ttf +0 -0
- sticker_convert/resources/help.ja_JP.json +88 -0
- sticker_convert/resources/help.json +10 -7
- sticker_convert/resources/help.zh_CN.json +88 -0
- sticker_convert/resources/help.zh_TW.json +88 -0
- sticker_convert/resources/input.ja_JP.json +74 -0
- sticker_convert/resources/input.json +121 -121
- sticker_convert/resources/input.zh_CN.json +74 -0
- sticker_convert/resources/input.zh_TW.json +74 -0
- sticker_convert/resources/output.ja_JP.json +38 -0
- sticker_convert/resources/output.zh_CN.json +38 -0
- sticker_convert/resources/output.zh_TW.json +38 -0
- sticker_convert/uploaders/compress_wastickers.py +186 -177
- sticker_convert/uploaders/upload_base.py +44 -35
- sticker_convert/uploaders/upload_signal.py +218 -203
- sticker_convert/uploaders/upload_telegram.py +353 -338
- sticker_convert/uploaders/upload_viber.py +178 -169
- sticker_convert/uploaders/xcode_imessage.py +295 -286
- sticker_convert/utils/callback.py +238 -6
- sticker_convert/utils/emoji.py +16 -4
- sticker_convert/utils/files/json_resources_loader.py +24 -19
- sticker_convert/utils/files/metadata_handler.py +3 -3
- sticker_convert/utils/translate.py +108 -0
- sticker_convert/utils/url_detect.py +40 -37
- sticker_convert/version.py +1 -1
- {sticker_convert-2.13.3.0.dist-info → sticker_convert-2.17.0.0.dist-info}/METADATA +89 -74
- {sticker_convert-2.13.3.0.dist-info → sticker_convert-2.17.0.0.dist-info}/RECORD +91 -74
- sticker_convert/utils/auth/telethon_setup.py +0 -97
- sticker_convert/utils/singletons.py +0 -18
- {sticker_convert-2.13.3.0.dist-info → sticker_convert-2.17.0.0.dist-info}/WHEEL +0 -0
- {sticker_convert-2.13.3.0.dist-info → sticker_convert-2.17.0.0.dist-info}/entry_points.txt +0 -0
- {sticker_convert-2.13.3.0.dist-info → sticker_convert-2.17.0.0.dist-info}/licenses/LICENSE +0 -0
- {sticker_convert-2.13.3.0.dist-info → sticker_convert-2.17.0.0.dist-info}/top_level.txt +0 -0
sticker_convert/definitions.py
CHANGED
|
@@ -3,6 +3,7 @@ import os
|
|
|
3
3
|
import platform
|
|
4
4
|
import sys
|
|
5
5
|
from pathlib import Path
|
|
6
|
+
from typing import Any, Dict
|
|
6
7
|
|
|
7
8
|
|
|
8
9
|
def get_root_dir() -> Path:
|
|
@@ -89,3 +90,6 @@ SVG_SAMPLE_FPS = 30
|
|
|
89
90
|
# If width and height not set in SVG tag, import at this dimension
|
|
90
91
|
SVG_DEFAULT_WIDTH = 1024
|
|
91
92
|
SVG_DEFAULT_HEIGHT = 1024
|
|
93
|
+
|
|
94
|
+
# A shared dictionary
|
|
95
|
+
RUNTIME_STATE: Dict[str, Any] = {}
|
|
@@ -1,110 +1,111 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
import json
|
|
5
|
-
import zipfile
|
|
6
|
-
from io import BytesIO
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
from typing import Any, Dict, Optional, Tuple
|
|
9
|
-
from urllib.parse import urlparse
|
|
10
|
-
|
|
11
|
-
import requests
|
|
12
|
-
|
|
13
|
-
from sticker_convert.downloaders.download_base import DownloadBase
|
|
14
|
-
from sticker_convert.job_option import CredOption, InputOption
|
|
15
|
-
from sticker_convert.utils.callback import CallbackProtocol, CallbackReturn
|
|
16
|
-
from sticker_convert.utils.files.metadata_handler import MetadataHandler
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import zipfile
|
|
6
|
+
from io import BytesIO
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, Optional, Tuple
|
|
9
|
+
from urllib.parse import urlparse
|
|
10
|
+
|
|
11
|
+
import requests
|
|
12
|
+
|
|
13
|
+
from sticker_convert.downloaders.download_base import DownloadBase
|
|
14
|
+
from sticker_convert.job_option import CredOption, InputOption
|
|
15
|
+
from sticker_convert.utils.callback import CallbackProtocol, CallbackReturn
|
|
16
|
+
from sticker_convert.utils.files.metadata_handler import MetadataHandler
|
|
17
|
+
from sticker_convert.utils.translate import I
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class DownloadBand(DownloadBase):
|
|
21
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
22
|
+
super().__init__(*args, **kwargs)
|
|
23
|
+
|
|
24
|
+
def decompress(self, zip_file: bytes) -> int:
|
|
25
|
+
with zipfile.ZipFile(BytesIO(zip_file)) as zf:
|
|
26
|
+
self.cb.put(I("Unzipping..."))
|
|
27
|
+
|
|
28
|
+
zf_files = zf.namelist()
|
|
29
|
+
animated = [i for i in zf_files if "animation/" in i]
|
|
30
|
+
if len(animated) > 0:
|
|
31
|
+
pack_files = animated
|
|
32
|
+
else:
|
|
33
|
+
pack_files = [
|
|
34
|
+
i
|
|
35
|
+
for i in zf_files
|
|
36
|
+
if i.endswith(".meta") is False and "_key" not in i
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
self.cb.put(
|
|
40
|
+
(
|
|
41
|
+
"bar",
|
|
42
|
+
None,
|
|
43
|
+
{"set_progress_mode": "determinate", "steps": len(pack_files)},
|
|
44
|
+
)
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
for num, sticker in enumerate(pack_files):
|
|
48
|
+
data = zf.read(sticker)
|
|
49
|
+
self.cb.put(f"Read {sticker}")
|
|
50
|
+
ext = sticker.split(".")[-1]
|
|
51
|
+
|
|
52
|
+
out_path = Path(self.out_dir, str(num).zfill(3) + f".{ext}")
|
|
53
|
+
with open(out_path, "wb") as f:
|
|
54
|
+
f.write(data)
|
|
55
|
+
|
|
56
|
+
self.cb.put("update_bar")
|
|
57
|
+
|
|
58
|
+
return len(pack_files)
|
|
59
|
+
|
|
60
|
+
def get_metadata(self, pack_no: str) -> Optional[Dict[Any, Any]]:
|
|
61
|
+
r = requests.get(
|
|
62
|
+
f"https://sapi.band.us/v1.0.0/get_sticker_info?pack_no={pack_no}"
|
|
63
|
+
)
|
|
64
|
+
if r.text:
|
|
65
|
+
return json.loads(r.text)
|
|
66
|
+
else:
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
def download_stickers_band(self) -> Tuple[int, int]:
|
|
70
|
+
if urlparse(self.url).netloc != "www.band.us" and self.url.isnumeric() is False:
|
|
71
|
+
self.cb.put(I("Download failed: Unsupported URL format"))
|
|
72
|
+
return 0, 0
|
|
73
|
+
|
|
74
|
+
if self.url.isnumeric():
|
|
75
|
+
pack_no = self.url
|
|
76
|
+
else:
|
|
77
|
+
pack_no = urlparse(self.url).path.split("/")[-1]
|
|
78
|
+
metadata = self.get_metadata(pack_no)
|
|
79
|
+
if metadata:
|
|
80
|
+
self.title = metadata["result_data"]["sticker"]["name"]
|
|
81
|
+
else:
|
|
82
|
+
self.cb.put(I("Download failed: Failed to get metadata"))
|
|
83
|
+
return 0, 0
|
|
84
|
+
|
|
85
|
+
MetadataHandler.set_metadata(self.out_dir, title=self.title)
|
|
86
|
+
|
|
87
|
+
pack_url = f"http://s.cmstatic.net/band/sticker/v2/{pack_no}/shop/pack"
|
|
88
|
+
zip_file = self.download_file(pack_url)
|
|
89
|
+
|
|
90
|
+
if zip_file:
|
|
91
|
+
self.cb.put(I("Downloaded {}").format(pack_url))
|
|
92
|
+
else:
|
|
93
|
+
self.cb.put(I("Cannot download {}").format(pack_url))
|
|
94
|
+
return 0, 0
|
|
95
|
+
|
|
96
|
+
pack_files_no = self.decompress(zip_file)
|
|
97
|
+
|
|
98
|
+
cover_url = f"http://s.cmstatic.net/band/sticker/v2/{pack_no}/shop/main"
|
|
99
|
+
self.download_file(cover_url, self.out_dir / "cover.png")
|
|
100
|
+
|
|
101
|
+
return pack_files_no, pack_files_no
|
|
102
|
+
|
|
103
|
+
@staticmethod
|
|
104
|
+
def start(
|
|
105
|
+
opt_input: InputOption,
|
|
106
|
+
opt_cred: Optional[CredOption],
|
|
107
|
+
cb: CallbackProtocol,
|
|
108
|
+
cb_return: CallbackReturn,
|
|
109
|
+
) -> Tuple[int, int]:
|
|
110
|
+
downloader = DownloadBand(opt_input, opt_cred, cb, cb_return)
|
|
111
|
+
return downloader.download_stickers_band()
|
|
@@ -1,166 +1,171 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
from functools import partial
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from typing import Any, Dict, List, Optional, Tuple
|
|
7
|
-
|
|
8
|
-
import anyio
|
|
9
|
-
import httpx
|
|
10
|
-
import requests
|
|
11
|
-
|
|
12
|
-
from sticker_convert.job_option import CredOption, InputOption
|
|
13
|
-
from sticker_convert.utils.callback import CallbackProtocol, CallbackReturn
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
self.
|
|
26
|
-
self.
|
|
27
|
-
self.
|
|
28
|
-
self.
|
|
29
|
-
self.
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
return b""
|
|
166
|
-
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from functools import partial
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
7
|
+
|
|
8
|
+
import anyio
|
|
9
|
+
import httpx
|
|
10
|
+
import requests
|
|
11
|
+
|
|
12
|
+
from sticker_convert.job_option import CredOption, InputOption
|
|
13
|
+
from sticker_convert.utils.callback import CallbackProtocol, CallbackReturn
|
|
14
|
+
from sticker_convert.utils.translate import I
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DownloadBase:
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
opt_input: InputOption,
|
|
21
|
+
opt_cred: Optional[CredOption],
|
|
22
|
+
cb: CallbackProtocol,
|
|
23
|
+
cb_return: CallbackReturn,
|
|
24
|
+
) -> None:
|
|
25
|
+
self.url = opt_input.url
|
|
26
|
+
self.out_dir = opt_input.dir
|
|
27
|
+
self.input_option = opt_input.option
|
|
28
|
+
self.opt_cred = opt_cred
|
|
29
|
+
self.cb = cb
|
|
30
|
+
self.cb_return = cb_return
|
|
31
|
+
|
|
32
|
+
def download_multiple_files(
|
|
33
|
+
self,
|
|
34
|
+
targets: List[Tuple[str, Path]],
|
|
35
|
+
retries: int = 3,
|
|
36
|
+
headers: Optional[dict[Any, Any]] = None,
|
|
37
|
+
**kwargs: Any,
|
|
38
|
+
) -> Dict[str, bool]:
|
|
39
|
+
results: Dict[str, bool] = {}
|
|
40
|
+
anyio.run(
|
|
41
|
+
partial(
|
|
42
|
+
self.download_multiple_files_async,
|
|
43
|
+
targets,
|
|
44
|
+
retries,
|
|
45
|
+
headers,
|
|
46
|
+
results,
|
|
47
|
+
**kwargs,
|
|
48
|
+
)
|
|
49
|
+
)
|
|
50
|
+
return results
|
|
51
|
+
|
|
52
|
+
async def download_multiple_files_async(
|
|
53
|
+
self,
|
|
54
|
+
targets: List[Tuple[str, Path]],
|
|
55
|
+
retries: int = 3,
|
|
56
|
+
headers: Optional[dict[Any, Any]] = None,
|
|
57
|
+
results: Optional[dict[str, bool]] = None,
|
|
58
|
+
**kwargs: Any,
|
|
59
|
+
) -> None:
|
|
60
|
+
# targets format: [(url1, dest2), (url2, dest2), ...]
|
|
61
|
+
self.cb.put(
|
|
62
|
+
("bar", None, {"set_progress_mode": "determinate", "steps": len(targets)})
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
semaphore = anyio.Semaphore(4)
|
|
66
|
+
|
|
67
|
+
async with httpx.AsyncClient() as client:
|
|
68
|
+
async with anyio.create_task_group() as tg:
|
|
69
|
+
for url, dest in targets:
|
|
70
|
+
tg.start_soon(
|
|
71
|
+
self.download_file_async,
|
|
72
|
+
semaphore,
|
|
73
|
+
client,
|
|
74
|
+
url,
|
|
75
|
+
dest,
|
|
76
|
+
retries,
|
|
77
|
+
headers,
|
|
78
|
+
results,
|
|
79
|
+
**kwargs,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
async def download_file_async(
|
|
83
|
+
self,
|
|
84
|
+
semaphore: anyio.Semaphore,
|
|
85
|
+
client: httpx.AsyncClient,
|
|
86
|
+
url: str,
|
|
87
|
+
dest: Path,
|
|
88
|
+
retries: int = 3,
|
|
89
|
+
headers: Optional[dict[Any, Any]] = None,
|
|
90
|
+
results: Optional[dict[str, bool]] = None,
|
|
91
|
+
**kwargs: Any,
|
|
92
|
+
) -> None:
|
|
93
|
+
async with semaphore:
|
|
94
|
+
self.cb.put(I("Downloading {}").format(url))
|
|
95
|
+
success = False
|
|
96
|
+
for retry in range(retries):
|
|
97
|
+
response = await client.get(
|
|
98
|
+
url, follow_redirects=True, headers=headers, **kwargs
|
|
99
|
+
)
|
|
100
|
+
success = response.is_success
|
|
101
|
+
|
|
102
|
+
if success:
|
|
103
|
+
async with await anyio.open_file(dest, "wb+") as f:
|
|
104
|
+
await f.write(response.content)
|
|
105
|
+
self.cb.put(I("Downloaded {}").format(url))
|
|
106
|
+
else:
|
|
107
|
+
self.cb.put(
|
|
108
|
+
I("Error {}: {} (tried {}/{} times)").format(
|
|
109
|
+
response.status_code, url, retry + 1, retries
|
|
110
|
+
)
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
if results is not None:
|
|
114
|
+
results[url] = success
|
|
115
|
+
|
|
116
|
+
self.cb.put("update_bar")
|
|
117
|
+
|
|
118
|
+
def download_file(
|
|
119
|
+
self,
|
|
120
|
+
url: str,
|
|
121
|
+
dest: Optional[Path] = None,
|
|
122
|
+
retries: int = 3,
|
|
123
|
+
show_progress: bool = True,
|
|
124
|
+
**kwargs: Any,
|
|
125
|
+
) -> bytes:
|
|
126
|
+
result = b""
|
|
127
|
+
chunk_size = 102400
|
|
128
|
+
|
|
129
|
+
for retry in range(retries):
|
|
130
|
+
try:
|
|
131
|
+
response = requests.get(
|
|
132
|
+
url, stream=True, allow_redirects=True, **kwargs
|
|
133
|
+
)
|
|
134
|
+
if not response.ok:
|
|
135
|
+
return b""
|
|
136
|
+
total_length = int(response.headers.get("content-length")) # type: ignore
|
|
137
|
+
|
|
138
|
+
self.cb.put(I("Downloading {}").format(url))
|
|
139
|
+
|
|
140
|
+
if show_progress:
|
|
141
|
+
steps = (total_length / chunk_size) + 1
|
|
142
|
+
self.cb.put(
|
|
143
|
+
(
|
|
144
|
+
"bar",
|
|
145
|
+
None,
|
|
146
|
+
{"set_progress_mode": "determinate", "steps": int(steps)},
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
for chunk in response.iter_content(chunk_size=chunk_size):
|
|
151
|
+
if chunk:
|
|
152
|
+
result += chunk
|
|
153
|
+
if show_progress:
|
|
154
|
+
self.cb.put("update_bar")
|
|
155
|
+
|
|
156
|
+
break
|
|
157
|
+
except requests.exceptions.RequestException as e:
|
|
158
|
+
self.cb.put(
|
|
159
|
+
I("Cannot download {} (tried {}/{} times): {}").format(
|
|
160
|
+
url, retry + 1, retries, e
|
|
161
|
+
)
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
if not result:
|
|
165
|
+
return b""
|
|
166
|
+
if dest:
|
|
167
|
+
with open(dest, "wb+") as f:
|
|
168
|
+
f.write(result)
|
|
169
|
+
self.cb.put(I("Downloaded {}").format(url))
|
|
170
|
+
return b""
|
|
171
|
+
return result
|