mangapy 3.0.0__tar.gz → 3.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mangapy-3.0.0 → mangapy-3.0.2}/PKG-INFO +23 -1
- {mangapy-3.0.0 → mangapy-3.0.2}/README.md +21 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/chapter_archiver.py +78 -11
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/cli.py +8 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/download_manager.py +34 -4
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/fanfox.py +31 -8
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/mangadex.py +27 -6
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy.egg-info/PKG-INFO +23 -1
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy.egg-info/requires.txt +1 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/pyproject.toml +2 -1
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_cli.py +5 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_download_manager.py +1 -1
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_fanfox_search.py +1 -1
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_mangadex.py +12 -2
- {mangapy-3.0.0 → mangapy-3.0.2}/LICENSE +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/__init__.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/capabilities.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/mangarepository.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/providers.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy/utils.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy.egg-info/SOURCES.txt +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy.egg-info/dependency_links.txt +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy.egg-info/entry_points.txt +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/mangapy.egg-info/top_level.txt +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/setup.cfg +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_chapter_archiver.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_fanfox.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_fanfox_packed_js.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_providers.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_sort_key.py +0 -0
- {mangapy-3.0.0 → mangapy-3.0.2}/tests/test_thread_safety.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mangapy
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.2
|
|
4
4
|
Summary: Manga downloader
|
|
5
5
|
Author-email: Alessandro Marzoli <alessandromarzoli@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -37,6 +37,7 @@ Requires-Python: >=3.11
|
|
|
37
37
|
Description-Content-Type: text/markdown
|
|
38
38
|
License-File: LICENSE
|
|
39
39
|
Requires-Dist: bs4
|
|
40
|
+
Requires-Dist: img2pdf<0.4
|
|
40
41
|
Requires-Dist: pillow
|
|
41
42
|
Requires-Dist: pyyaml
|
|
42
43
|
Requires-Dist: requests
|
|
@@ -56,6 +57,12 @@ Manga downloader supporting the following sources:
|
|
|
56
57
|
pipx install mangapy
|
|
57
58
|
```
|
|
58
59
|
|
|
60
|
+
Or, on macOS with Homebrew:
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
brew install alemar11/tap/mangapy
|
|
64
|
+
```
|
|
65
|
+
|
|
59
66
|
## Usage
|
|
60
67
|
|
|
61
68
|
### Terminal
|
|
@@ -87,6 +94,18 @@ Downloads Bleach chatpers from 0 to 10 (included) as images inside the *Download
|
|
|
87
94
|
mangapy title bleach -c 0-10 -o ~/Downloads -s fanfox
|
|
88
95
|
```
|
|
89
96
|
|
|
97
|
+
Disable network retries (useful for benchmarking).
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
mangapy title bleach -c 1 -o ~/Downloads --no-retry
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Disable progress output.
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
mangapy title bleach -c 1 -o ~/Downloads --no-progress
|
|
107
|
+
```
|
|
108
|
+
|
|
90
109
|
You may need a proxy to download certain manga, to do so use the option *-p or --proxy*:
|
|
91
110
|
Downloads the last One Piece chapter as images inside the *Downloads* folder (from Fanfox source) using the proxy during the search.
|
|
92
111
|
|
|
@@ -116,6 +135,8 @@ uv run python3 scripts/dev_run.py <sample-filename.yaml>
|
|
|
116
135
|
```yaml
|
|
117
136
|
---
|
|
118
137
|
debug: true # optional
|
|
138
|
+
no_retry: false # optional, disable retries
|
|
139
|
+
no_progress: false # optional, disable progress output
|
|
119
140
|
output: "~/Downloads/mangapy"
|
|
120
141
|
proxy: # optional
|
|
121
142
|
http: "http://31.14.131.70:8080"
|
|
@@ -125,6 +146,7 @@ uv run python3 scripts/dev_run.py <sample-filename.yaml>
|
|
|
125
146
|
title: "bleach"
|
|
126
147
|
pdf: true
|
|
127
148
|
download_single_chapter: "10"
|
|
149
|
+
no_retry: true
|
|
128
150
|
- source: "fanfox"
|
|
129
151
|
title: "naruto"
|
|
130
152
|
pdf: true
|
|
@@ -11,6 +11,12 @@ Manga downloader supporting the following sources:
|
|
|
11
11
|
pipx install mangapy
|
|
12
12
|
```
|
|
13
13
|
|
|
14
|
+
Or, on macOS with Homebrew:
|
|
15
|
+
|
|
16
|
+
```
|
|
17
|
+
brew install alemar11/tap/mangapy
|
|
18
|
+
```
|
|
19
|
+
|
|
14
20
|
## Usage
|
|
15
21
|
|
|
16
22
|
### Terminal
|
|
@@ -42,6 +48,18 @@ Downloads Bleach chatpers from 0 to 10 (included) as images inside the *Download
|
|
|
42
48
|
mangapy title bleach -c 0-10 -o ~/Downloads -s fanfox
|
|
43
49
|
```
|
|
44
50
|
|
|
51
|
+
Disable network retries (useful for benchmarking).
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
mangapy title bleach -c 1 -o ~/Downloads --no-retry
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Disable progress output.
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
mangapy title bleach -c 1 -o ~/Downloads --no-progress
|
|
61
|
+
```
|
|
62
|
+
|
|
45
63
|
You may need a proxy to download certain manga, to do so use the option *-p or --proxy*:
|
|
46
64
|
Downloads the last One Piece chapter as images inside the *Downloads* folder (from Fanfox source) using the proxy during the search.
|
|
47
65
|
|
|
@@ -71,6 +89,8 @@ uv run python3 scripts/dev_run.py <sample-filename.yaml>
|
|
|
71
89
|
```yaml
|
|
72
90
|
---
|
|
73
91
|
debug: true # optional
|
|
92
|
+
no_retry: false # optional, disable retries
|
|
93
|
+
no_progress: false # optional, disable progress output
|
|
74
94
|
output: "~/Downloads/mangapy"
|
|
75
95
|
proxy: # optional
|
|
76
96
|
http: "http://31.14.131.70:8080"
|
|
@@ -80,6 +100,7 @@ uv run python3 scripts/dev_run.py <sample-filename.yaml>
|
|
|
80
100
|
title: "bleach"
|
|
81
101
|
pdf: true
|
|
82
102
|
download_single_chapter: "10"
|
|
103
|
+
no_retry: true
|
|
83
104
|
- source: "fanfox"
|
|
84
105
|
title: "naruto"
|
|
85
106
|
pdf: true
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
|
+
import random
|
|
3
4
|
import re
|
|
5
|
+
import sys
|
|
4
6
|
import requests
|
|
5
7
|
import shutil
|
|
6
8
|
import threading
|
|
9
|
+
import time
|
|
7
10
|
from collections.abc import Mapping
|
|
8
11
|
from functools import partial
|
|
9
12
|
from tqdm import tqdm
|
|
@@ -18,9 +21,11 @@ tqdm.set_lock(threading.RLock())
|
|
|
18
21
|
|
|
19
22
|
|
|
20
23
|
class ChapterArchiver(object):
|
|
21
|
-
def __init__(self, path: str, max_workers=1):
|
|
24
|
+
def __init__(self, path: str, max_workers=1, retry_enabled: bool = True, show_progress: bool = True):
|
|
22
25
|
self.max_workers = max_workers
|
|
23
26
|
self.path = Path(path).expanduser()
|
|
27
|
+
self.retry_enabled = retry_enabled
|
|
28
|
+
self.show_progress = show_progress
|
|
24
29
|
self._session_local = threading.local()
|
|
25
30
|
self._pdf_lock_guard = threading.Lock()
|
|
26
31
|
self._pdf_locks = {}
|
|
@@ -51,10 +56,35 @@ class ChapterArchiver(object):
|
|
|
51
56
|
|
|
52
57
|
def _fetch_image(self, url: str, headers: Mapping[str, str | bytes | None] | None):
|
|
53
58
|
session = self._get_session()
|
|
54
|
-
|
|
55
|
-
|
|
59
|
+
if not self.retry_enabled:
|
|
60
|
+
try:
|
|
61
|
+
response = session.get(url, headers=headers, timeout=(10, 30))
|
|
62
|
+
except requests.RequestException as exc:
|
|
63
|
+
logging.error("Failed to download image %s: %s", url, exc)
|
|
64
|
+
return None
|
|
65
|
+
if response.status_code != 200:
|
|
66
|
+
return None
|
|
67
|
+
return response.content
|
|
68
|
+
|
|
69
|
+
last_error = None
|
|
70
|
+
for attempt in range(3):
|
|
71
|
+
try:
|
|
72
|
+
response = session.get(url, headers=headers, timeout=(10, 30))
|
|
73
|
+
except requests.RequestException as exc:
|
|
74
|
+
last_error = exc
|
|
75
|
+
time.sleep(_retry_delay(attempt))
|
|
76
|
+
continue
|
|
77
|
+
|
|
78
|
+
if response.status_code == 200:
|
|
79
|
+
return response.content
|
|
80
|
+
if response.status_code == 429 or response.status_code >= 500:
|
|
81
|
+
time.sleep(_retry_delay(attempt, response))
|
|
82
|
+
continue
|
|
56
83
|
return None
|
|
57
|
-
|
|
84
|
+
|
|
85
|
+
if last_error:
|
|
86
|
+
logging.error("Failed to download image %s: %s", url, last_error)
|
|
87
|
+
return None
|
|
58
88
|
|
|
59
89
|
def _save_image(self, image_path: Path, headers: Mapping[str, str | bytes | None] | None, page: Page):
|
|
60
90
|
file_name = str(page.number)
|
|
@@ -65,7 +95,7 @@ class ChapterArchiver(object):
|
|
|
65
95
|
return # early exit if the file is already on disk
|
|
66
96
|
|
|
67
97
|
if image_url.startswith('//'):
|
|
68
|
-
image_url = '
|
|
98
|
+
image_url = 'https:' + image_url
|
|
69
99
|
|
|
70
100
|
data = self._fetch_image(image_url, headers=headers)
|
|
71
101
|
|
|
@@ -73,15 +103,19 @@ class ChapterArchiver(object):
|
|
|
73
103
|
logging.error("Can't download page {0}".format(file_name))
|
|
74
104
|
return
|
|
75
105
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
output.close()
|
|
106
|
+
with open(file_path, "wb") as output:
|
|
107
|
+
output.write(data)
|
|
79
108
|
|
|
80
109
|
def _create_chapter_pdf(self, chapter_images_path: Path, pdf_path: Path):
|
|
81
|
-
file_list =
|
|
110
|
+
file_list = [path for path in chapter_images_path.glob('**/*') if path.is_file()]
|
|
82
111
|
chapter_images_path = list(map(lambda path: str(path.absolute()), file_list))
|
|
83
112
|
images_path = natural_sort(chapter_images_path)
|
|
84
113
|
|
|
114
|
+
tmp_path = pdf_path.with_name(pdf_path.name + ".tmp")
|
|
115
|
+
if _try_img2pdf(images_path, tmp_path):
|
|
116
|
+
tmp_path.replace(pdf_path)
|
|
117
|
+
return
|
|
118
|
+
|
|
85
119
|
images = []
|
|
86
120
|
for path in images_path:
|
|
87
121
|
image = Image.open(path)
|
|
@@ -94,7 +128,6 @@ class ChapterArchiver(object):
|
|
|
94
128
|
if images_count <= 0:
|
|
95
129
|
return
|
|
96
130
|
first_image = images.pop(0)
|
|
97
|
-
tmp_path = pdf_path.with_name(pdf_path.name + ".tmp")
|
|
98
131
|
try:
|
|
99
132
|
if images_count == 1:
|
|
100
133
|
first_image.save(tmp_path, "PDF", resolution=100.0, save_all=True)
|
|
@@ -147,8 +180,16 @@ class ChapterArchiver(object):
|
|
|
147
180
|
description = ('Chapter {0}'.format(chapter_name))
|
|
148
181
|
func = partial(self._save_image, chapter_images_path, headers) # currying
|
|
149
182
|
|
|
183
|
+
disable_progress = (not self.show_progress) or (not sys.stderr.isatty())
|
|
150
184
|
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
|
151
|
-
for _ in tqdm(
|
|
185
|
+
for _ in tqdm(
|
|
186
|
+
executor.map(func, pages),
|
|
187
|
+
total=len(pages),
|
|
188
|
+
desc=description,
|
|
189
|
+
unit='pages',
|
|
190
|
+
ncols=100,
|
|
191
|
+
disable=disable_progress,
|
|
192
|
+
):
|
|
152
193
|
pass
|
|
153
194
|
|
|
154
195
|
|
|
@@ -160,3 +201,29 @@ def natural_sort(list):
|
|
|
160
201
|
return [convert(c) for c in re.split('([0-9]+)', key)]
|
|
161
202
|
|
|
162
203
|
return sorted(list, key=alphanum_key)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _retry_delay(attempt: int, response: requests.Response | None = None) -> float:
|
|
207
|
+
if response is not None:
|
|
208
|
+
retry_after = response.headers.get("Retry-After")
|
|
209
|
+
if retry_after and retry_after.isdigit():
|
|
210
|
+
return float(retry_after)
|
|
211
|
+
base = min(2 ** attempt, 5)
|
|
212
|
+
return base + random.uniform(0, 0.2)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _try_img2pdf(image_paths: list[str], tmp_path: Path) -> bool:
|
|
216
|
+
if not image_paths:
|
|
217
|
+
return False
|
|
218
|
+
try:
|
|
219
|
+
import img2pdf
|
|
220
|
+
except Exception:
|
|
221
|
+
return False
|
|
222
|
+
|
|
223
|
+
try:
|
|
224
|
+
with open(tmp_path, "wb") as output:
|
|
225
|
+
output.write(img2pdf.convert(image_paths, dpi=100))
|
|
226
|
+
return True
|
|
227
|
+
except Exception as exc:
|
|
228
|
+
logging.warning("img2pdf failed, falling back to PIL: %s", exc)
|
|
229
|
+
return False
|
|
@@ -30,6 +30,8 @@ def cmd_parse():
|
|
|
30
30
|
args_parser.add_argument('-o', '--out', type=str, default=default_path_to_download_folder, help='download directory')
|
|
31
31
|
args_parser.add_argument('-d', '--debug', action='store_true', help="set log to DEBUG level")
|
|
32
32
|
args_parser.add_argument('--pdf', action='store_true', help="create a pdf for each chapter")
|
|
33
|
+
args_parser.add_argument('--no-retry', action='store_true', help="disable network retries")
|
|
34
|
+
args_parser.add_argument('--no-progress', action='store_true', help="disable progress output")
|
|
33
35
|
|
|
34
36
|
args_parser.add_argument('-p', '--proxy', type=json.loads, help="use a proxy to download the chapters")
|
|
35
37
|
group = args_parser.add_mutually_exclusive_group()
|
|
@@ -75,6 +77,8 @@ def main_yaml(args: argparse.Namespace):
|
|
|
75
77
|
print('The proxy is not in the right format and it will not be used.')
|
|
76
78
|
|
|
77
79
|
debug_log = bool(dictionary.get('debug', False))
|
|
80
|
+
no_retry = bool(dictionary.get('no_retry', False))
|
|
81
|
+
no_progress = bool(dictionary.get('no_progress', False))
|
|
78
82
|
downloads = _normalize_yaml_downloads(dictionary)
|
|
79
83
|
manager = DownloadManager()
|
|
80
84
|
for entry in downloads:
|
|
@@ -87,6 +91,8 @@ def main_yaml(args: argparse.Namespace):
|
|
|
87
91
|
output=str(entry.get('output', output)).strip(),
|
|
88
92
|
pdf=bool(entry.get('pdf', False)),
|
|
89
93
|
proxy=entry.get('proxy', proxy),
|
|
94
|
+
no_retry=bool(entry.get('no_retry', no_retry)),
|
|
95
|
+
no_progress=bool(entry.get('no_progress', no_progress)),
|
|
90
96
|
enable_debug_log=bool(entry.get('debug', debug_log)),
|
|
91
97
|
download_all_chapters=bool(entry.get('download_all_chapters', False)),
|
|
92
98
|
download_last_chapter=bool(entry.get('download_last_chapter', False)),
|
|
@@ -115,6 +121,8 @@ def main_title(args: argparse.Namespace):
|
|
|
115
121
|
output=args.out.strip(),
|
|
116
122
|
pdf=args.pdf or False,
|
|
117
123
|
proxy=proxy,
|
|
124
|
+
no_retry=bool(getattr(args, "no_retry", False)),
|
|
125
|
+
no_progress=bool(getattr(args, "no_progress", False)),
|
|
118
126
|
enable_debug_log=args.debug,
|
|
119
127
|
download_all_chapters=bool(args.all),
|
|
120
128
|
download_single_chapter=_parse_single_chapter(args.chapter),
|
|
@@ -17,6 +17,8 @@ class DownloadRequest:
|
|
|
17
17
|
output: str
|
|
18
18
|
pdf: bool = False
|
|
19
19
|
proxy: dict | None = None
|
|
20
|
+
no_retry: bool = False
|
|
21
|
+
no_progress: bool = False
|
|
20
22
|
enable_debug_log: bool = False
|
|
21
23
|
download_all_chapters: bool = False
|
|
22
24
|
download_last_chapter: bool = False
|
|
@@ -35,6 +37,8 @@ class DownloadManager:
|
|
|
35
37
|
repository = get_repository(request.source)
|
|
36
38
|
if request.proxy:
|
|
37
39
|
repository.proxies = request.proxy
|
|
40
|
+
if hasattr(repository, "no_retry"):
|
|
41
|
+
repository.no_retry = request.no_retry
|
|
38
42
|
|
|
39
43
|
print(f"🔎 Searching for {request.title} in {request.source}...")
|
|
40
44
|
try:
|
|
@@ -85,20 +89,46 @@ class DownloadManager:
|
|
|
85
89
|
with ThreadPoolExecutor(max_workers=repository.capabilities.max_parallel_chapters) as executor:
|
|
86
90
|
list(
|
|
87
91
|
executor.map(
|
|
88
|
-
lambda ch: _archive_chapter(
|
|
92
|
+
lambda ch: _archive_chapter(
|
|
93
|
+
directory,
|
|
94
|
+
max_parallel_pages,
|
|
95
|
+
ch,
|
|
96
|
+
request.pdf,
|
|
97
|
+
headers,
|
|
98
|
+
retry_enabled=not request.no_retry,
|
|
99
|
+
show_progress=not request.no_progress,
|
|
100
|
+
),
|
|
89
101
|
chapters,
|
|
90
102
|
)
|
|
91
103
|
)
|
|
92
104
|
else:
|
|
93
|
-
archiver = ChapterArchiver(
|
|
105
|
+
archiver = ChapterArchiver(
|
|
106
|
+
directory,
|
|
107
|
+
max_workers=max_parallel_pages,
|
|
108
|
+
retry_enabled=not request.no_retry,
|
|
109
|
+
show_progress=not request.no_progress,
|
|
110
|
+
)
|
|
94
111
|
for chapter in chapters:
|
|
95
112
|
_archive_with_archiver(archiver, chapter, request.pdf, headers)
|
|
96
113
|
|
|
97
114
|
print("🎉 Download finished.")
|
|
98
115
|
|
|
99
116
|
|
|
100
|
-
def _archive_chapter(
|
|
101
|
-
|
|
117
|
+
def _archive_chapter(
|
|
118
|
+
directory: str,
|
|
119
|
+
max_parallel_pages: int,
|
|
120
|
+
chapter: Chapter,
|
|
121
|
+
pdf: bool,
|
|
122
|
+
headers,
|
|
123
|
+
retry_enabled: bool,
|
|
124
|
+
show_progress: bool,
|
|
125
|
+
):
|
|
126
|
+
archiver = ChapterArchiver(
|
|
127
|
+
directory,
|
|
128
|
+
max_workers=max_parallel_pages,
|
|
129
|
+
retry_enabled=retry_enabled,
|
|
130
|
+
show_progress=show_progress,
|
|
131
|
+
)
|
|
102
132
|
_archive_with_archiver(archiver, chapter, pdf, headers)
|
|
103
133
|
|
|
104
134
|
|
|
@@ -77,7 +77,7 @@ class FanFoxRepository(MangaRepository):
|
|
|
77
77
|
return manga_chapters
|
|
78
78
|
chapters = list_detail.find_all('a', href=True)
|
|
79
79
|
chapters_url = map(lambda c: c['href'], reversed(chapters))
|
|
80
|
-
|
|
80
|
+
|
|
81
81
|
for url in chapters_url:
|
|
82
82
|
chapter_id = url.split("/")[-2][1:] # relative url, TODO: regex
|
|
83
83
|
absolute_url = "{0}{1}".format(self.base_url, url)
|
|
@@ -109,7 +109,10 @@ class FanFoxRepository(MangaRepository):
|
|
|
109
109
|
return self._fetch_manga(match_url, match_title)
|
|
110
110
|
|
|
111
111
|
def _fetch_manga(self, manga_url: str, fallback_title: str) -> Manga | None:
|
|
112
|
-
|
|
112
|
+
try:
|
|
113
|
+
response = self.session.get(url=manga_url, timeout=(10, 30))
|
|
114
|
+
except requests.RequestException:
|
|
115
|
+
return None
|
|
113
116
|
if response is None or response.status_code != 200:
|
|
114
117
|
return None
|
|
115
118
|
|
|
@@ -150,7 +153,10 @@ class FanFoxRepository(MangaRepository):
|
|
|
150
153
|
return Manga(fallback_title, sorted_chapters)
|
|
151
154
|
|
|
152
155
|
def _search_manga(self, manga_name: str) -> tuple[str, str] | None:
|
|
153
|
-
|
|
156
|
+
try:
|
|
157
|
+
response = self.session.get(f"{self.base_url}/search", params={"title": manga_name}, timeout=(10, 30))
|
|
158
|
+
except requests.RequestException:
|
|
159
|
+
return None
|
|
154
160
|
if response is None or response.status_code != 200:
|
|
155
161
|
return None
|
|
156
162
|
|
|
@@ -201,6 +207,8 @@ class FanFoxChapter(Chapter):
|
|
|
201
207
|
if unpacked is None:
|
|
202
208
|
raise ValueError("Packed JS not found for key")
|
|
203
209
|
key_match = re.search(r'(?<=var guidkey=)(.*)(?=\';)', unpacked)
|
|
210
|
+
if not key_match:
|
|
211
|
+
return ""
|
|
204
212
|
key = key_match.group(1)
|
|
205
213
|
key = key.replace('\'', '')
|
|
206
214
|
key = key.replace('\\', '')
|
|
@@ -225,7 +233,10 @@ class FanFoxChapter(Chapter):
|
|
|
225
233
|
def _one_link_helper(self, page, base_url, cid, key):
|
|
226
234
|
final_url = '{}/chapterfun.ashx?cid={}&page={}&key={}'.format(base_url, cid, page, key)
|
|
227
235
|
session = self._session_provider()
|
|
228
|
-
|
|
236
|
+
try:
|
|
237
|
+
response = session.get(final_url, timeout=(10, 30))
|
|
238
|
+
except requests.RequestException:
|
|
239
|
+
return None
|
|
229
240
|
|
|
230
241
|
if response is None or response.status_code != 200:
|
|
231
242
|
return None
|
|
@@ -234,14 +245,20 @@ class FanFoxChapter(Chapter):
|
|
|
234
245
|
return content
|
|
235
246
|
|
|
236
247
|
def _parse_links(self, data):
|
|
237
|
-
|
|
248
|
+
base_match = re.search(r'pix="(.+?)"', data)
|
|
249
|
+
if not base_match:
|
|
250
|
+
return []
|
|
251
|
+
base_path = base_match.group(1)
|
|
238
252
|
images = re.findall(r'"(/\w.+?)"', data)
|
|
239
253
|
return [base_path + i for i in images]
|
|
240
254
|
|
|
241
255
|
def pages(self) -> List[Page]:
|
|
242
256
|
base_url = self.first_page_url[:self.first_page_url.rfind('/')]
|
|
243
257
|
session = self._session_provider()
|
|
244
|
-
|
|
258
|
+
try:
|
|
259
|
+
response = session.get(self.first_page_url, timeout=(10, 30))
|
|
260
|
+
except requests.RequestException:
|
|
261
|
+
return None
|
|
245
262
|
|
|
246
263
|
if response is None or response.status_code != 200:
|
|
247
264
|
return None
|
|
@@ -275,9 +292,15 @@ class FanFoxChapter(Chapter):
|
|
|
275
292
|
scripts = soup.find_all("script", {"type": "text/javascript"})
|
|
276
293
|
if not scripts:
|
|
277
294
|
return []
|
|
278
|
-
|
|
295
|
+
eval_node = next((script for script in scripts if script.text.startswith("eval")), None)
|
|
296
|
+
if not eval_node or not eval_node.text:
|
|
297
|
+
return []
|
|
298
|
+
eval_script = eval_node.text
|
|
279
299
|
images_content = self._get_urls(eval_script)
|
|
280
|
-
|
|
300
|
+
links_match = re.search(r'(?<=newImgs=\[)[^]]+(?=\])', images_content)
|
|
301
|
+
if not links_match:
|
|
302
|
+
return []
|
|
303
|
+
links = links_match.group(0).split(',')
|
|
281
304
|
for i, link in enumerate(links):
|
|
282
305
|
formatted_link = link.replace("'", "")
|
|
283
306
|
pages.append(Page(i, formatted_link))
|
|
@@ -14,6 +14,7 @@ class MangadexRepository(MangaRepository):
|
|
|
14
14
|
self._session_local = threading.local()
|
|
15
15
|
self._rate_lock = threading.Lock()
|
|
16
16
|
self._last_request = 0.0
|
|
17
|
+
self.no_retry = False
|
|
17
18
|
|
|
18
19
|
@property
|
|
19
20
|
def capabilities(self) -> ProviderCapabilities:
|
|
@@ -121,14 +122,29 @@ class MangadexRepository(MangaRepository):
|
|
|
121
122
|
|
|
122
123
|
def _request(self, url: str, params: dict | None = None) -> requests.Response | None:
|
|
123
124
|
session = self._get_session()
|
|
125
|
+
if self.no_retry:
|
|
126
|
+
self._apply_rate_limit()
|
|
127
|
+
try:
|
|
128
|
+
return session.get(url, params=params, timeout=(10, 30))
|
|
129
|
+
except requests.RequestException:
|
|
130
|
+
return None
|
|
131
|
+
last_error = None
|
|
124
132
|
for attempt in range(3):
|
|
125
133
|
self._apply_rate_limit()
|
|
126
|
-
|
|
134
|
+
try:
|
|
135
|
+
response = session.get(url, params=params, timeout=(10, 30))
|
|
136
|
+
except requests.RequestException as exc:
|
|
137
|
+
last_error = exc
|
|
138
|
+
delay = _retry_delay(None, attempt)
|
|
139
|
+
time.sleep(delay)
|
|
140
|
+
continue
|
|
127
141
|
if response.status_code == 429 or response.status_code >= 500:
|
|
128
142
|
delay = _retry_delay(response, attempt)
|
|
129
143
|
time.sleep(delay)
|
|
130
144
|
continue
|
|
131
145
|
return response
|
|
146
|
+
if last_error:
|
|
147
|
+
return None
|
|
132
148
|
return response
|
|
133
149
|
|
|
134
150
|
def _apply_rate_limit(self) -> None:
|
|
@@ -180,7 +196,11 @@ class MangadexChapter(Chapter):
|
|
|
180
196
|
if not self.chapter_uuid:
|
|
181
197
|
return []
|
|
182
198
|
if self._requester is None:
|
|
183
|
-
response = requests.get(
|
|
199
|
+
response = requests.get(
|
|
200
|
+
self.first_page_url,
|
|
201
|
+
timeout=(10, 30),
|
|
202
|
+
headers={"Accept": "application/json", "User-Agent": "mangapy"},
|
|
203
|
+
)
|
|
184
204
|
if response.status_code != 200:
|
|
185
205
|
return []
|
|
186
206
|
payload = response.json()
|
|
@@ -261,8 +281,9 @@ def _chapter_sort_key(volume: str | None, chapter: str | None, chapter_id: str):
|
|
|
261
281
|
return (2, chapter_id)
|
|
262
282
|
|
|
263
283
|
|
|
264
|
-
def _retry_delay(response: requests.Response, attempt: int) -> float:
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
284
|
+
def _retry_delay(response: requests.Response | None, attempt: int) -> float:
|
|
285
|
+
if response is not None:
|
|
286
|
+
retry_after = response.headers.get("Retry-After")
|
|
287
|
+
if retry_after and retry_after.isdigit():
|
|
288
|
+
return float(retry_after)
|
|
268
289
|
return min(2 ** attempt, 5)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mangapy
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.2
|
|
4
4
|
Summary: Manga downloader
|
|
5
5
|
Author-email: Alessandro Marzoli <alessandromarzoli@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -37,6 +37,7 @@ Requires-Python: >=3.11
|
|
|
37
37
|
Description-Content-Type: text/markdown
|
|
38
38
|
License-File: LICENSE
|
|
39
39
|
Requires-Dist: bs4
|
|
40
|
+
Requires-Dist: img2pdf<0.4
|
|
40
41
|
Requires-Dist: pillow
|
|
41
42
|
Requires-Dist: pyyaml
|
|
42
43
|
Requires-Dist: requests
|
|
@@ -56,6 +57,12 @@ Manga downloader supporting the following sources:
|
|
|
56
57
|
pipx install mangapy
|
|
57
58
|
```
|
|
58
59
|
|
|
60
|
+
Or, on macOS with Homebrew:
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
brew install alemar11/tap/mangapy
|
|
64
|
+
```
|
|
65
|
+
|
|
59
66
|
## Usage
|
|
60
67
|
|
|
61
68
|
### Terminal
|
|
@@ -87,6 +94,18 @@ Downloads Bleach chatpers from 0 to 10 (included) as images inside the *Download
|
|
|
87
94
|
mangapy title bleach -c 0-10 -o ~/Downloads -s fanfox
|
|
88
95
|
```
|
|
89
96
|
|
|
97
|
+
Disable network retries (useful for benchmarking).
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
mangapy title bleach -c 1 -o ~/Downloads --no-retry
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Disable progress output.
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
mangapy title bleach -c 1 -o ~/Downloads --no-progress
|
|
107
|
+
```
|
|
108
|
+
|
|
90
109
|
You may need a proxy to download certain manga, to do so use the option *-p or --proxy*:
|
|
91
110
|
Downloads the last One Piece chapter as images inside the *Downloads* folder (from Fanfox source) using the proxy during the search.
|
|
92
111
|
|
|
@@ -116,6 +135,8 @@ uv run python3 scripts/dev_run.py <sample-filename.yaml>
|
|
|
116
135
|
```yaml
|
|
117
136
|
---
|
|
118
137
|
debug: true # optional
|
|
138
|
+
no_retry: false # optional, disable retries
|
|
139
|
+
no_progress: false # optional, disable progress output
|
|
119
140
|
output: "~/Downloads/mangapy"
|
|
120
141
|
proxy: # optional
|
|
121
142
|
http: "http://31.14.131.70:8080"
|
|
@@ -125,6 +146,7 @@ uv run python3 scripts/dev_run.py <sample-filename.yaml>
|
|
|
125
146
|
title: "bleach"
|
|
126
147
|
pdf: true
|
|
127
148
|
download_single_chapter: "10"
|
|
149
|
+
no_retry: true
|
|
128
150
|
- source: "fanfox"
|
|
129
151
|
title: "naruto"
|
|
130
152
|
pdf: true
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "mangapy"
|
|
7
|
-
version = "3.0.
|
|
7
|
+
version = "3.0.2"
|
|
8
8
|
description = "Manga downloader"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -22,6 +22,7 @@ classifiers = [
|
|
|
22
22
|
]
|
|
23
23
|
dependencies = [
|
|
24
24
|
"bs4",
|
|
25
|
+
"img2pdf<0.4",
|
|
25
26
|
"pillow",
|
|
26
27
|
"pyyaml",
|
|
27
28
|
"requests",
|
|
@@ -70,6 +70,7 @@ def test_main_title_accepts_valid_proxy(monkeypatch):
|
|
|
70
70
|
proxy={"http": "h", "https": "s"},
|
|
71
71
|
all=False,
|
|
72
72
|
chapter=None,
|
|
73
|
+
no_progress=False,
|
|
73
74
|
)
|
|
74
75
|
|
|
75
76
|
cli.main_title(args)
|
|
@@ -88,6 +89,7 @@ def test_main_title_rejects_proxy_missing_http(monkeypatch):
|
|
|
88
89
|
proxy={"https": "s"},
|
|
89
90
|
all=False,
|
|
90
91
|
chapter=None,
|
|
92
|
+
no_progress=False,
|
|
91
93
|
)
|
|
92
94
|
|
|
93
95
|
cli.main_title(args)
|
|
@@ -106,6 +108,7 @@ def test_main_title_source_and_range(monkeypatch):
|
|
|
106
108
|
proxy=None,
|
|
107
109
|
all=False,
|
|
108
110
|
chapter="1-3",
|
|
111
|
+
no_progress=False,
|
|
109
112
|
)
|
|
110
113
|
|
|
111
114
|
cli.main_title(args)
|
|
@@ -169,6 +172,7 @@ def test_main_yaml_downloads_list(monkeypatch, tmp_path):
|
|
|
169
172
|
"translated_language": ["it"],
|
|
170
173
|
"content_rating": "safe",
|
|
171
174
|
"data_saver": True,
|
|
175
|
+
"no_progress": True,
|
|
172
176
|
},
|
|
173
177
|
{"title": ""},
|
|
174
178
|
{
|
|
@@ -192,6 +196,7 @@ def test_main_yaml_downloads_list(monkeypatch, tmp_path):
|
|
|
192
196
|
assert first.output == "/tmp/root"
|
|
193
197
|
assert first.proxy == {"http": "h", "https": "s"}
|
|
194
198
|
assert first.download_all_chapters is True
|
|
199
|
+
assert first.no_progress is True
|
|
195
200
|
assert first.options == {
|
|
196
201
|
"translated_language": ["it"],
|
|
197
202
|
"content_rating": "safe",
|
|
@@ -33,7 +33,7 @@ def test_download_manager_parallel_chapters(monkeypatch):
|
|
|
33
33
|
|
|
34
34
|
monkeypatch.setattr("mangapy.download_manager.get_repository", lambda name: repo)
|
|
35
35
|
|
|
36
|
-
def fake_archive(directory, max_parallel_pages, chapter, pdf, headers):
|
|
36
|
+
def fake_archive(directory, max_parallel_pages, chapter, pdf, headers, **kwargs):
|
|
37
37
|
calls.append(chapter.chapter_id)
|
|
38
38
|
|
|
39
39
|
monkeypatch.setattr("mangapy.download_manager._archive_chapter", fake_archive)
|
|
@@ -13,7 +13,7 @@ class _FakeSession:
|
|
|
13
13
|
def __init__(self, responses: dict[str, list[_FakeResponse]]):
|
|
14
14
|
self._responses = {key: list(values) for key, values in responses.items()}
|
|
15
15
|
|
|
16
|
-
def get(self, url, params=None):
|
|
16
|
+
def get(self, url, params=None, **kwargs):
|
|
17
17
|
key = url
|
|
18
18
|
if key not in self._responses:
|
|
19
19
|
if "search" in url:
|
|
@@ -5,14 +5,24 @@ from mangapy.mangadex import MangadexChapter, MangadexRepository
|
|
|
5
5
|
|
|
6
6
|
def _latest_en_chapter():
|
|
7
7
|
url = "https://api.mangadex.org/chapter"
|
|
8
|
-
params = {"translatedLanguage[]": ["en"], "order[readableAt]": "desc", "limit":
|
|
8
|
+
params = {"translatedLanguage[]": ["en"], "order[readableAt]": "desc", "limit": 10}
|
|
9
9
|
response = requests.get(url, params=params, timeout=(10, 30))
|
|
10
10
|
response.raise_for_status()
|
|
11
11
|
payload = response.json()
|
|
12
12
|
data = payload.get("data", [])
|
|
13
13
|
if not data:
|
|
14
14
|
return None
|
|
15
|
-
chapter =
|
|
15
|
+
chapter = None
|
|
16
|
+
for candidate in data:
|
|
17
|
+
attributes = candidate.get("attributes", {})
|
|
18
|
+
if attributes.get("externalUrl"):
|
|
19
|
+
continue
|
|
20
|
+
if (attributes.get("pages") or 0) <= 0:
|
|
21
|
+
continue
|
|
22
|
+
chapter = candidate
|
|
23
|
+
break
|
|
24
|
+
if chapter is None:
|
|
25
|
+
return None
|
|
16
26
|
manga_id = None
|
|
17
27
|
for rel in chapter.get("relationships", []):
|
|
18
28
|
if rel.get("type") == "manga":
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|