novel-downloader 1.1.1__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- novel_downloader/__init__.py +1 -1
- novel_downloader/cli/download.py +58 -24
- novel_downloader/config/adapter.py +4 -1
- novel_downloader/config/models.py +4 -1
- novel_downloader/core/downloaders/__init__.py +2 -0
- novel_downloader/core/downloaders/base_async_downloader.py +157 -0
- novel_downloader/core/downloaders/common_asynb_downloader.py +207 -0
- novel_downloader/core/downloaders/common_downloader.py +2 -3
- novel_downloader/core/factory/__init__.py +14 -2
- novel_downloader/core/factory/downloader_factory.py +95 -8
- novel_downloader/core/factory/requester_factory.py +65 -21
- novel_downloader/core/interfaces/__init__.py +4 -0
- novel_downloader/core/interfaces/async_downloader_protocol.py +37 -0
- novel_downloader/core/interfaces/async_requester_protocol.py +68 -0
- novel_downloader/core/requesters/__init__.py +5 -1
- novel_downloader/core/requesters/base_async_session.py +297 -0
- novel_downloader/core/requesters/common_requester/__init__.py +5 -1
- novel_downloader/core/requesters/common_requester/common_async_session.py +96 -0
- novel_downloader/core/requesters/qidian_requester/qidian_session.py +1 -1
- novel_downloader/resources/config/settings.yaml +4 -1
- novel_downloader/utils/crypto_utils.py +4 -4
- {novel_downloader-1.1.1.dist-info → novel_downloader-1.2.0.dist-info}/METADATA +27 -7
- {novel_downloader-1.1.1.dist-info → novel_downloader-1.2.0.dist-info}/RECORD +27 -21
- {novel_downloader-1.1.1.dist-info → novel_downloader-1.2.0.dist-info}/WHEEL +0 -0
- {novel_downloader-1.1.1.dist-info → novel_downloader-1.2.0.dist-info}/entry_points.txt +0 -0
- {novel_downloader-1.1.1.dist-info → novel_downloader-1.2.0.dist-info}/licenses/LICENSE +0 -0
- {novel_downloader-1.1.1.dist-info → novel_downloader-1.2.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,297 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
"""
|
4
|
+
novel_downloader.core.requesters.base_async_session
|
5
|
+
---------------------------------------------------
|
6
|
+
|
7
|
+
This module defines the BaseAsyncSession class, which provides asynchronous
|
8
|
+
HTTP request capabilities using aiohttp. It maintains a persistent
|
9
|
+
client session and supports retries, headers, timeout configurations,
|
10
|
+
cookie handling, and defines abstract methods for subclasses.
|
11
|
+
"""
|
12
|
+
|
13
|
+
import abc
|
14
|
+
import asyncio
|
15
|
+
import time
|
16
|
+
from typing import Any, Dict, Optional, Union
|
17
|
+
|
18
|
+
import aiohttp
|
19
|
+
from aiohttp import ClientResponse, ClientSession, ClientTimeout, TCPConnector
|
20
|
+
|
21
|
+
from novel_downloader.config.models import RequesterConfig
|
22
|
+
from novel_downloader.core.interfaces import AsyncRequesterProtocol
|
23
|
+
from novel_downloader.utils.constants import DEFAULT_USER_HEADERS
|
24
|
+
|
25
|
+
|
26
|
+
class RateLimiter:
|
27
|
+
"""
|
28
|
+
Simple async token-bucket rate limiter: ensures no more than rate_per_sec
|
29
|
+
requests are started per second, across all coroutines.
|
30
|
+
"""
|
31
|
+
|
32
|
+
def __init__(self, rate_per_sec: float):
|
33
|
+
self._interval = 1.0 / rate_per_sec
|
34
|
+
self._lock = asyncio.Lock()
|
35
|
+
self._last = time.monotonic()
|
36
|
+
|
37
|
+
async def wait(self) -> None:
|
38
|
+
async with self._lock:
|
39
|
+
now = time.monotonic()
|
40
|
+
elapsed = now - self._last
|
41
|
+
delay = self._interval - elapsed
|
42
|
+
if delay > 0:
|
43
|
+
await asyncio.sleep(delay)
|
44
|
+
self._last = time.monotonic()
|
45
|
+
|
46
|
+
|
47
|
+
class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
|
48
|
+
"""
|
49
|
+
BaseAsyncSession wraps basic HTTP operations using aiohttp.ClientSession,
|
50
|
+
supporting retry logic, timeout, persistent connections, and cookie management.
|
51
|
+
|
52
|
+
Attributes:
|
53
|
+
_session (ClientSession): The persistent aiohttp client session.
|
54
|
+
_timeout (int): Timeout for each request in seconds.
|
55
|
+
_retry_times (int): Number of retry attempts on failure.
|
56
|
+
_retry_interval (float): Delay (in seconds) between retries.
|
57
|
+
_headers (Dict[str, str]): Default HTTP headers to send.
|
58
|
+
_cookies (Dict[str, str]): Optional cookie jar for the session.
|
59
|
+
"""
|
60
|
+
|
61
|
+
def _init_session(
|
62
|
+
self,
|
63
|
+
config: RequesterConfig,
|
64
|
+
cookies: Optional[Dict[str, str]] = None,
|
65
|
+
) -> None:
|
66
|
+
"""
|
67
|
+
Initialize the async session with configuration.
|
68
|
+
|
69
|
+
:param config: Configuration object for session behavior
|
70
|
+
(timeouts, retries, headers, etc.)
|
71
|
+
:param cookies: Optional initial cookies to set on the session.
|
72
|
+
"""
|
73
|
+
self._config = config
|
74
|
+
self._timeout = config.timeout
|
75
|
+
self._retry_times = config.retry_times
|
76
|
+
self._retry_interval = config.retry_interval
|
77
|
+
self._cookies = cookies or {}
|
78
|
+
self._headers = DEFAULT_USER_HEADERS.copy()
|
79
|
+
self._session: Optional[ClientSession] = None
|
80
|
+
self._rate_limiter: Optional[RateLimiter] = None
|
81
|
+
|
82
|
+
async def _setup(self) -> None:
|
83
|
+
"""
|
84
|
+
Set up the aiohttp.ClientSession with timeout, connector, headers, and cookies.
|
85
|
+
"""
|
86
|
+
max_rps = getattr(self._config, "max_rps", None)
|
87
|
+
if max_rps is not None:
|
88
|
+
self._rate_limiter = RateLimiter(max_rps)
|
89
|
+
|
90
|
+
timeout = ClientTimeout(total=self._timeout)
|
91
|
+
connector = TCPConnector(
|
92
|
+
limit_per_host=getattr(self._config, "max_connections", 10)
|
93
|
+
)
|
94
|
+
self._session = ClientSession(
|
95
|
+
timeout=timeout,
|
96
|
+
connector=connector,
|
97
|
+
headers=self._headers,
|
98
|
+
cookies=self._cookies,
|
99
|
+
)
|
100
|
+
|
101
|
+
async def login(self, max_retries: int = 3, manual_login: bool = False) -> bool:
|
102
|
+
"""
|
103
|
+
Attempt to log in asynchronously.
|
104
|
+
Override in subclasses that require authentication.
|
105
|
+
|
106
|
+
:returns: True if login succeeded, False otherwise.
|
107
|
+
"""
|
108
|
+
raise NotImplementedError(
|
109
|
+
"Login is not supported by this session type. "
|
110
|
+
"Override login() in your subclass to enable it."
|
111
|
+
)
|
112
|
+
|
113
|
+
@abc.abstractmethod
|
114
|
+
async def get_book_info(self, book_id: str, wait_time: Optional[int] = None) -> str:
|
115
|
+
"""
|
116
|
+
Fetch the raw HTML (or JSON) of the book info page asynchronously.
|
117
|
+
|
118
|
+
:param book_id: The book identifier.
|
119
|
+
:param wait_time: Base number of seconds to wait before returning content.
|
120
|
+
:return: The page content as a string.
|
121
|
+
"""
|
122
|
+
...
|
123
|
+
|
124
|
+
@abc.abstractmethod
|
125
|
+
async def get_book_chapter(
|
126
|
+
self, book_id: str, chapter_id: str, wait_time: Optional[int] = None
|
127
|
+
) -> str:
|
128
|
+
"""
|
129
|
+
Fetch the raw HTML (or JSON) of a single chapter asynchronously.
|
130
|
+
|
131
|
+
:param book_id: The book identifier.
|
132
|
+
:param chapter_id: The chapter identifier.
|
133
|
+
:param wait_time: Base number of seconds to wait before returning content.
|
134
|
+
:return: The chapter content as a string.
|
135
|
+
"""
|
136
|
+
...
|
137
|
+
|
138
|
+
async def get_bookcase(self, wait_time: Optional[int] = None) -> str:
|
139
|
+
"""
|
140
|
+
Optional: Retrieve the HTML content of the authenticated user's bookcase page.
|
141
|
+
Subclasses that support user login/bookcase should override this.
|
142
|
+
|
143
|
+
:param wait_time: Base number of seconds to wait before returning content.
|
144
|
+
:return: The HTML of the bookcase page.
|
145
|
+
"""
|
146
|
+
raise NotImplementedError(
|
147
|
+
"Bookcase fetching is not supported by this session type. "
|
148
|
+
"Override get_bookcase() in your subclass to enable it."
|
149
|
+
)
|
150
|
+
|
151
|
+
async def fetch(self, url: str, **kwargs: Any) -> str:
|
152
|
+
"""
|
153
|
+
Fetch the content from the given URL asynchronously, with retry support.
|
154
|
+
|
155
|
+
:param url: The target URL to fetch.
|
156
|
+
:param kwargs: Additional keyword arguments to pass to `session.get`.
|
157
|
+
:return: The response body as text.
|
158
|
+
:raises: aiohttp.ClientError on final failure.
|
159
|
+
"""
|
160
|
+
if self._session is None:
|
161
|
+
await self._setup()
|
162
|
+
if self._session is None:
|
163
|
+
raise RuntimeError("Session not initialized after setup")
|
164
|
+
|
165
|
+
if self._rate_limiter:
|
166
|
+
await self._rate_limiter.wait()
|
167
|
+
|
168
|
+
for attempt in range(self._retry_times + 1):
|
169
|
+
try:
|
170
|
+
async with self._session.get(url, **kwargs) as resp:
|
171
|
+
resp.raise_for_status()
|
172
|
+
text: str = await resp.text()
|
173
|
+
return text
|
174
|
+
except aiohttp.ClientError:
|
175
|
+
if attempt < self._retry_times:
|
176
|
+
await asyncio.sleep(self._retry_interval)
|
177
|
+
continue
|
178
|
+
raise
|
179
|
+
|
180
|
+
raise RuntimeError("Unreachable code reached in fetch()")
|
181
|
+
|
182
|
+
async def get(
|
183
|
+
self, url: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
|
184
|
+
) -> ClientResponse:
|
185
|
+
"""
|
186
|
+
Send an HTTP GET request asynchronously.
|
187
|
+
|
188
|
+
:param url: The target URL.
|
189
|
+
:param params: Query parameters to include in the request.
|
190
|
+
:param kwargs: Additional args passed to session.get().
|
191
|
+
:return: aiohttp.ClientResponse object.
|
192
|
+
:raises RuntimeError: If the session is not initialized.
|
193
|
+
"""
|
194
|
+
if self._session is None:
|
195
|
+
await self._setup()
|
196
|
+
if self._session is None:
|
197
|
+
raise RuntimeError("Session not initialized after setup")
|
198
|
+
|
199
|
+
if self._rate_limiter:
|
200
|
+
await self._rate_limiter.wait()
|
201
|
+
return await self._session.get(url, params=params, **kwargs)
|
202
|
+
|
203
|
+
async def post(
|
204
|
+
self,
|
205
|
+
url: str,
|
206
|
+
data: Optional[Union[Dict[str, Any], bytes]] = None,
|
207
|
+
json: Optional[Dict[str, Any]] = None,
|
208
|
+
**kwargs: Any,
|
209
|
+
) -> ClientResponse:
|
210
|
+
"""
|
211
|
+
Send an HTTP POST request asynchronously.
|
212
|
+
|
213
|
+
:param url: The target URL.
|
214
|
+
:param data: Form data to include in the request body.
|
215
|
+
:param json: JSON body to include in the request.
|
216
|
+
:param kwargs: Additional args passed to session.post().
|
217
|
+
:return: aiohttp.ClientResponse object.
|
218
|
+
:raises RuntimeError: If the session is not initialized.
|
219
|
+
"""
|
220
|
+
if self._session is None:
|
221
|
+
await self._setup()
|
222
|
+
if self._session is None:
|
223
|
+
raise RuntimeError("Session not initialized after setup")
|
224
|
+
|
225
|
+
if self._rate_limiter:
|
226
|
+
await self._rate_limiter.wait()
|
227
|
+
return await self._session.post(url, data=data, json=json, **kwargs)
|
228
|
+
|
229
|
+
@property
|
230
|
+
def session(self) -> ClientSession:
|
231
|
+
"""
|
232
|
+
Return the active aiohttp.ClientSession.
|
233
|
+
|
234
|
+
:raises RuntimeError: If the session is uninitialized.
|
235
|
+
"""
|
236
|
+
if self._session is None:
|
237
|
+
raise RuntimeError("Session is not initialized or has been shut down.")
|
238
|
+
return self._session
|
239
|
+
|
240
|
+
@property
|
241
|
+
def timeout(self) -> int:
|
242
|
+
"""Return the default timeout setting."""
|
243
|
+
return self._timeout
|
244
|
+
|
245
|
+
@property
|
246
|
+
def retry_times(self) -> int:
|
247
|
+
"""Return the maximum number of retry attempts."""
|
248
|
+
return self._retry_times
|
249
|
+
|
250
|
+
@property
|
251
|
+
def retry_interval(self) -> float:
|
252
|
+
"""Return the base interval (in seconds) between retries."""
|
253
|
+
return self._retry_interval
|
254
|
+
|
255
|
+
async def update_cookies(
|
256
|
+
self, cookies: Dict[str, str], overwrite: bool = True
|
257
|
+
) -> None:
|
258
|
+
"""
|
259
|
+
Update cookies for the current session and internal cache.
|
260
|
+
|
261
|
+
:param cookies: New cookies to merge.
|
262
|
+
:param overwrite: If True, replace existing; else, only set missing.
|
263
|
+
"""
|
264
|
+
# update internal cache
|
265
|
+
if overwrite:
|
266
|
+
self._cookies.update({str(k): str(v) for k, v in cookies.items()})
|
267
|
+
else:
|
268
|
+
for k, v in cookies.items():
|
269
|
+
self._cookies.setdefault(str(k), str(v))
|
270
|
+
|
271
|
+
# apply to live session
|
272
|
+
if self._session:
|
273
|
+
self._session.cookie_jar.update_cookies(self._cookies)
|
274
|
+
|
275
|
+
async def shutdown(self) -> None:
|
276
|
+
"""
|
277
|
+
Shutdown and clean up the session. Closes connection pool.
|
278
|
+
"""
|
279
|
+
if self._session:
|
280
|
+
await self._session.close()
|
281
|
+
self._session = None
|
282
|
+
|
283
|
+
def __getstate__(self) -> Dict[str, Any]:
|
284
|
+
"""
|
285
|
+
Prepare object state for serialization: remove unpickleable session.
|
286
|
+
"""
|
287
|
+
state = self.__dict__.copy()
|
288
|
+
state.pop("_session", None)
|
289
|
+
state.pop("_rate_limiter", None)
|
290
|
+
return state
|
291
|
+
|
292
|
+
def __setstate__(self, state: Dict[str, Any]) -> None:
|
293
|
+
"""
|
294
|
+
Restore object state. Session will be lazily reinitialized on next request.
|
295
|
+
"""
|
296
|
+
self.__dict__.update(state)
|
297
|
+
self._session = None
|
@@ -9,6 +9,10 @@ request operations to novel websites. It serves as a unified access
|
|
9
9
|
point to import `CommonSession` without exposing lower-level modules.
|
10
10
|
"""
|
11
11
|
|
12
|
+
from .common_async_session import CommonAsyncSession
|
12
13
|
from .common_session import CommonSession
|
13
14
|
|
14
|
-
__all__ = [
|
15
|
+
__all__ = [
|
16
|
+
"CommonAsyncSession",
|
17
|
+
"CommonSession",
|
18
|
+
]
|
@@ -0,0 +1,96 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
"""
|
4
|
+
novel_downloader.core.requesters.common_requester.common_async_session
|
5
|
+
----------------------------------------------------------------------
|
6
|
+
|
7
|
+
This module defines a `CommonAsyncSession` class for handling HTTP requests
|
8
|
+
to common novel sites **asynchronously**. It provides methods to retrieve
|
9
|
+
raw book info pages and chapter contents using a flexible URL templating
|
10
|
+
system defined by a site profile, with retry logic and random delays.
|
11
|
+
"""
|
12
|
+
|
13
|
+
import asyncio
|
14
|
+
import random
|
15
|
+
from typing import Dict, Optional
|
16
|
+
|
17
|
+
from novel_downloader.config import RequesterConfig, SiteProfile
|
18
|
+
from novel_downloader.core.requesters.base_async_session import BaseAsyncSession
|
19
|
+
|
20
|
+
|
21
|
+
class CommonAsyncSession(BaseAsyncSession):
|
22
|
+
"""
|
23
|
+
A common async session for handling site-specific HTTP requests.
|
24
|
+
|
25
|
+
:ivar _site: The unique identifier or name of the site.
|
26
|
+
:ivar _profile: Metadata and URL templates related to the site.
|
27
|
+
"""
|
28
|
+
|
29
|
+
def __init__(
|
30
|
+
self,
|
31
|
+
config: RequesterConfig,
|
32
|
+
site: str,
|
33
|
+
profile: SiteProfile,
|
34
|
+
cookies: Optional[Dict[str, str]] = None,
|
35
|
+
) -> None:
|
36
|
+
"""
|
37
|
+
Initialize a CommonAsyncSession instance.
|
38
|
+
|
39
|
+
:param config: The RequesterConfig instance containing settings.
|
40
|
+
:param site: The identifier or domain of the target site.
|
41
|
+
:param profile: The site's metadata and URL templates.
|
42
|
+
:param cookies: Optional cookies to preload into the session.
|
43
|
+
"""
|
44
|
+
self._init_session(config=config, cookies=cookies)
|
45
|
+
self._site = site
|
46
|
+
self._profile = profile
|
47
|
+
|
48
|
+
async def get_book_info(self, book_id: str, wait_time: Optional[int] = None) -> str:
|
49
|
+
"""
|
50
|
+
Fetch the raw HTML of the book info page asynchronously.
|
51
|
+
|
52
|
+
Relies on BaseAsyncSession.fetch for retry logic, then sleeps with jitter.
|
53
|
+
|
54
|
+
:param book_id: The book identifier.
|
55
|
+
:param wait_time: Base seconds to sleep (with 0.5-1.5x random factor).
|
56
|
+
:return: The page content as a string.
|
57
|
+
"""
|
58
|
+
url = self.book_info_url.format(book_id=book_id)
|
59
|
+
html = await self.fetch(url)
|
60
|
+
base = wait_time if wait_time is not None else self._config.wait_time
|
61
|
+
await asyncio.sleep(base * random.uniform(0.5, 1.5))
|
62
|
+
return html
|
63
|
+
|
64
|
+
async def get_book_chapter(
|
65
|
+
self, book_id: str, chapter_id: str, wait_time: Optional[int] = None
|
66
|
+
) -> str:
|
67
|
+
"""
|
68
|
+
Fetch the raw HTML of a single chapter asynchronously.
|
69
|
+
|
70
|
+
Relies on BaseAsyncSession.fetch for retry logic, then sleeps with jitter.
|
71
|
+
|
72
|
+
:param book_id: The book identifier.
|
73
|
+
:param chapter_id: The chapter identifier.
|
74
|
+
:param wait_time: Base seconds to sleep (with 0.5-1.5x random factor).
|
75
|
+
:return: The chapter content as a string.
|
76
|
+
"""
|
77
|
+
url = self.chapter_url.format(book_id=book_id, chapter_id=chapter_id)
|
78
|
+
html = await self.fetch(url)
|
79
|
+
base = wait_time if wait_time is not None else self._config.wait_time
|
80
|
+
await asyncio.sleep(base * random.uniform(0.5, 1.5))
|
81
|
+
return html
|
82
|
+
|
83
|
+
@property
|
84
|
+
def site(self) -> str:
|
85
|
+
"""Return the site name."""
|
86
|
+
return self._site
|
87
|
+
|
88
|
+
@property
|
89
|
+
def book_info_url(self) -> str:
|
90
|
+
"""Return the URL template for fetching book info."""
|
91
|
+
return self._profile["book_info_url"]
|
92
|
+
|
93
|
+
@property
|
94
|
+
def chapter_url(self) -> str:
|
95
|
+
"""Return the URL template for fetching chapter content."""
|
96
|
+
return self._profile["chapter_url"]
|
@@ -70,7 +70,7 @@ class QidianSession(BaseSession):
|
|
70
70
|
3. Updates both the live ``requests.Session`` and the internal cache;
|
71
71
|
4. Delegates the actual request to ``super().get``.
|
72
72
|
"""
|
73
|
-
if self._session is None: # defensive
|
73
|
+
if self._session is None: # defensive - mirrors BaseSession check
|
74
74
|
raise RuntimeError("Session is not initialized or has been shut down.")
|
75
75
|
|
76
76
|
# ---- 1. refresh token cookie --------------------------------------
|
@@ -4,6 +4,7 @@ requests:
|
|
4
4
|
retry_times: 3 # 请求失败重试次数
|
5
5
|
retry_interval: 5
|
6
6
|
timeout: 30 # 页面加载超时时间 (秒)
|
7
|
+
max_rps: null # 最大请求速率 (requests per second), 为 null 则不限制
|
7
8
|
# DrissionPage 专用设置
|
8
9
|
headless: false # 是否以无头模式启动浏览器
|
9
10
|
user_data_folder: "" # 浏览器用户数据目录: 为空则使用默认目录
|
@@ -18,7 +19,9 @@ general:
|
|
18
19
|
raw_data_dir: "./raw_data" # 原始章节 HTML/JSON 存放目录
|
19
20
|
output_dir: "./downloads" # 最终输出文件存放目录
|
20
21
|
cache_dir: "./novel_cache" # 本地缓存目录 (字体 / 图片等)
|
21
|
-
|
22
|
+
download_workers: 4 # 并发下载线程数
|
23
|
+
parser_workers: 4 # 并发解析线程数
|
24
|
+
use_process_pool: false # 是否使用多进程池来处理任务
|
22
25
|
skip_existing: true # 是否跳过已存在章节
|
23
26
|
debug:
|
24
27
|
save_html: false # 是否将抓取到的原始 HTML 保留到磁盘
|
@@ -108,17 +108,17 @@ def patch_qd_payload_token(
|
|
108
108
|
if not key:
|
109
109
|
key = _get_key()
|
110
110
|
|
111
|
-
# Step 1
|
111
|
+
# Step 1 - decrypt --------------------------------------------------
|
112
112
|
decrypted_json: str = rc4_crypt(key, enc_token, mode="decrypt")
|
113
113
|
payload: Dict[str, Any] = json.loads(decrypted_json)
|
114
114
|
|
115
|
-
# Step 2
|
115
|
+
# Step 2 - rebuild timing fields -----------------------------------
|
116
116
|
loadts = int(time.time() * 1000) # ms since epoch
|
117
117
|
# Simulate the JS duration: N(600, 150) pushed into [300, 1000]
|
118
118
|
duration = max(300, min(1000, int(random.normalvariate(600, 150))))
|
119
119
|
timestamp = loadts + duration
|
120
120
|
|
121
|
-
# Step 3
|
121
|
+
# Step 3 - recalculate ------------------------------------
|
122
122
|
fp_key = _d("ZmluZ2VycHJpbnQ=")
|
123
123
|
ab_key = _d("YWJub3JtYWw=")
|
124
124
|
ck_key = _d("Y2hlY2tzdW0=")
|
@@ -138,7 +138,7 @@ def patch_qd_payload_token(
|
|
138
138
|
ck_key: ck_val,
|
139
139
|
}
|
140
140
|
|
141
|
-
# Step 4
|
141
|
+
# Step 4 - encrypt and return --------------------------------------
|
142
142
|
return rc4_crypt(
|
143
143
|
key, json.dumps(new_payload, separators=(",", ":")), mode="encrypt"
|
144
144
|
)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: novel-downloader
|
3
|
-
Version: 1.
|
3
|
+
Version: 1.2.0
|
4
4
|
Summary: A command-line tool for downloading Chinese web novels from Qidian and similar platforms.
|
5
5
|
Author-email: Saudade Z <saudadez217@gmail.com>
|
6
6
|
License: MIT License
|
@@ -66,6 +66,8 @@ Requires-Dist: tinycss2; extra == "font-recovery"
|
|
66
66
|
Requires-Dist: fonttools; extra == "font-recovery"
|
67
67
|
Requires-Dist: pillow; extra == "font-recovery"
|
68
68
|
Requires-Dist: huggingface_hub; extra == "font-recovery"
|
69
|
+
Provides-Extra: async
|
70
|
+
Requires-Dist: aiohttp; extra == "async"
|
69
71
|
Dynamic: license-file
|
70
72
|
|
71
73
|
# novel-downloader
|
@@ -87,19 +89,37 @@ Dynamic: license-file
|
|
87
89
|
|
88
90
|
```bash
|
89
91
|
# 克隆 + 安装
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
#
|
92
|
+
pip install novel-downloader
|
93
|
+
|
94
|
+
# 如需支持字体解密功能 (decode_font), 请使用:
|
95
|
+
# pip install novel-downloader[font-recovery]
|
94
96
|
|
95
|
-
#
|
97
|
+
# 如需启用异步抓取模式 (mode=async), 请使用:
|
98
|
+
# pip install novel-downloader[async]
|
99
|
+
|
100
|
+
# 初始化默认配置 (生成 settings.yaml)
|
96
101
|
novel-cli settings init
|
97
102
|
|
98
|
-
# 编辑 ./settings.yaml 完成 site/book_ids
|
103
|
+
# 编辑 ./settings.yaml 完成 site/book_ids 等
|
104
|
+
# 可查看 docs/4-settings-schema.md
|
105
|
+
|
99
106
|
# 运行下载
|
100
107
|
novel-cli download 123456
|
101
108
|
```
|
102
109
|
|
110
|
+
**从 GitHub 安装 (开发版)**
|
111
|
+
|
112
|
+
如需体验开发中的最新功能, 可通过 GitHub 安装:
|
113
|
+
|
114
|
+
```bash
|
115
|
+
git clone https://github.com/BowenZ217/novel-downloader.git
|
116
|
+
cd novel-downloader
|
117
|
+
pip install .
|
118
|
+
# 或安装带可选功能:
|
119
|
+
# pip install .[font-recovery]
|
120
|
+
# pip install .[async]
|
121
|
+
```
|
122
|
+
|
103
123
|
更多使用方法, 查看 [使用示例](https://github.com/BowenZ217/novel-downloader/blob/main/docs/5-usage-examples.md)
|
104
124
|
|
105
125
|
---
|
@@ -1,26 +1,30 @@
|
|
1
|
-
novel_downloader/__init__.py,sha256=
|
1
|
+
novel_downloader/__init__.py,sha256=y5cwfjeZ2r8jpzNfl3w7_Vc4R9CzZWLbPe599UjiX7g,242
|
2
2
|
novel_downloader/cli/__init__.py,sha256=ocGwOO4kmkby8VNol92UikMI1RPUJLv9i5xmB7wbpmw,198
|
3
3
|
novel_downloader/cli/clean.py,sha256=9_hOrxKg8nY7q6cyR8iNech0vSREGagPBmdB4k8Te2U,3937
|
4
|
-
novel_downloader/cli/download.py,sha256=
|
4
|
+
novel_downloader/cli/download.py,sha256=l-Ht2duKI78EMR8vTEbdVnwFT9NkWe87l3L1LmmIuZc,4156
|
5
5
|
novel_downloader/cli/interactive.py,sha256=6vROwPsvupb_TWH1dd_78FDqvtAaiPfyEBvQVai9E9c,2154
|
6
6
|
novel_downloader/cli/main.py,sha256=km1MwHzIVZFcxUlKLRiiMctJlGHWKZNjRKrgAGQjkMs,1183
|
7
7
|
novel_downloader/cli/settings.py,sha256=bV3Hgg502V9goeP3g2xSiF-PMQB9G32qGmjb8ncTENA,6522
|
8
8
|
novel_downloader/config/__init__.py,sha256=tJ2k7nwZbxgqw1kKgJM4g1yu5-2fsx2LXU3VTadrTJ4,1129
|
9
|
-
novel_downloader/config/adapter.py,sha256=
|
9
|
+
novel_downloader/config/adapter.py,sha256=wZgkNWDjGuPev1wMmLnIfZQnc4UT-MWfsIWn_PwQWuw,5807
|
10
10
|
novel_downloader/config/loader.py,sha256=_rm9rp1lmHYg-A7F_0PQETWjlXbvtyJYaqQD5oI-1O0,5690
|
11
|
-
novel_downloader/config/models.py,sha256=
|
11
|
+
novel_downloader/config/models.py,sha256=8cWXG0SIu3E0jTMlFdOHI9E-lFils0nUqyfmvxJmCn8,5015
|
12
12
|
novel_downloader/config/site_rules.py,sha256=WRw12Tfue-ErAPGKq506gRIqKOxWU-u96kay3JDgTNc,3031
|
13
13
|
novel_downloader/core/__init__.py,sha256=D-ACiIqP0rdARZmjBnF6WMKGvvjVtxGRIM7GhOS9kh4,779
|
14
|
-
novel_downloader/core/downloaders/__init__.py,sha256=
|
14
|
+
novel_downloader/core/downloaders/__init__.py,sha256=Qp0q4p7zTy7lReQQF0hDP7ALUQnNflSNNIl4F7iPGz0,601
|
15
|
+
novel_downloader/core/downloaders/base_async_downloader.py,sha256=7IDbPeeJhIx9h5iaTO5KuAkPdvkLghIQXCqIJXiCGKY,4279
|
15
16
|
novel_downloader/core/downloaders/base_downloader.py,sha256=kFw_yn3QRbWqU9jXJni4IGA8P3AxZf9gfjgfu01TauY,5371
|
16
|
-
novel_downloader/core/downloaders/
|
17
|
+
novel_downloader/core/downloaders/common_asynb_downloader.py,sha256=u1ODvh_n13CSGWwjkBIMoThTbCeACX5mOhv5ub2Cd0c,7120
|
18
|
+
novel_downloader/core/downloaders/common_downloader.py,sha256=Ru60j-S9I-Nj1P7gNZJjohJ1H8gAuvK1bELPMeZ2TTo,6532
|
17
19
|
novel_downloader/core/downloaders/qidian_downloader.py,sha256=Btt59d8N925gx3RqTN7rqaHhMQgLxzPtOhE4Iq6wl1k,7354
|
18
|
-
novel_downloader/core/factory/__init__.py,sha256=
|
19
|
-
novel_downloader/core/factory/downloader_factory.py,sha256=
|
20
|
+
novel_downloader/core/factory/__init__.py,sha256=qGqeROj8Dp_5iNtgWytkrUNI0ICab7SCNK3lba3H_NU,743
|
21
|
+
novel_downloader/core/factory/downloader_factory.py,sha256=54lgfJ8KDJWYlBfWb6iWniRe55K9bPkcTTRVtipH7nU,5047
|
20
22
|
novel_downloader/core/factory/parser_factory.py,sha256=4PxiagtSKY58azFsmEWfq2f5vhVbtMFm5gAXS3oQF08,1828
|
21
|
-
novel_downloader/core/factory/requester_factory.py,sha256=
|
23
|
+
novel_downloader/core/factory/requester_factory.py,sha256=OEK2S-rj8vw4IdDTMTEWcb7k7lRmmWBnfijhYnlOCc0,3173
|
22
24
|
novel_downloader/core/factory/saver_factory.py,sha256=OgZPDOWVIfhxLFiVBKI5jaNOEKmzP9f3YWDOnw63Hfc,1275
|
23
|
-
novel_downloader/core/interfaces/__init__.py,sha256=
|
25
|
+
novel_downloader/core/interfaces/__init__.py,sha256=jeT8BmEEjIazVyX80ZdzQXgTccEj-ktG6Bbjs9uAVUM,843
|
26
|
+
novel_downloader/core/interfaces/async_downloader_protocol.py,sha256=QWjdhNc39hC3bD8Q1lUpBv2GqX3roxVxzKWh6cgwLhk,1002
|
27
|
+
novel_downloader/core/interfaces/async_requester_protocol.py,sha256=pOFto57lkO6o_eT0eSsheO6XXPDJ-NG4CQuTftp3luY,2261
|
24
28
|
novel_downloader/core/interfaces/downloader_protocol.py,sha256=YJdSAE9uBWF7wNLWmlKsYd3J7M4rXOjqDV5m9O7kfio,947
|
25
29
|
novel_downloader/core/interfaces/parser_protocol.py,sha256=A2wIe7shEGdeKsNDFpMuPI8HFrK_H34HOseVAzqcnTo,1280
|
26
30
|
novel_downloader/core/interfaces/requester_protocol.py,sha256=jXeVh-cO8Euv1T59P4pzYzKxdxu-sneZlUMfzj8A2qs,1988
|
@@ -45,14 +49,16 @@ novel_downloader/core/parsers/qidian_parser/session/node_decryptor.py,sha256=7Zu
|
|
45
49
|
novel_downloader/core/parsers/qidian_parser/shared/__init__.py,sha256=K5HX7pgiRiJuTLdbQDbtm60mO-sXgr6bo5Ft8H1-JLs,978
|
46
50
|
novel_downloader/core/parsers/qidian_parser/shared/book_info_parser.py,sha256=juCV72QKcaAjQZU-j6XiBM1VgdRrXY9w_2NHrflHsv4,3047
|
47
51
|
novel_downloader/core/parsers/qidian_parser/shared/helpers.py,sha256=E8cWVhehaMLNXQAq2whIKl29xAULUzW4MdZvWshDb9Y,4284
|
48
|
-
novel_downloader/core/requesters/__init__.py,sha256=
|
52
|
+
novel_downloader/core/requesters/__init__.py,sha256=U2jDvt5RxF5P9yL2lwyZ-cRueJBZgRnjil3_5TvAh3Y,798
|
53
|
+
novel_downloader/core/requesters/base_async_session.py,sha256=kN2-m2vbMzF0F321VsYSDWIOU7na_TNNF-r0E6cGMi0,10540
|
49
54
|
novel_downloader/core/requesters/base_browser.py,sha256=anAR9QfbNllhX1TbzRXevWZfaURDN7s6GlPSoqvoZ8M,6753
|
50
55
|
novel_downloader/core/requesters/base_session.py,sha256=keQ-QFr1JS8Zkypv28X_jZ44ecRemUZSuructHLgypc,8245
|
51
|
-
novel_downloader/core/requesters/common_requester/__init__.py,sha256=
|
56
|
+
novel_downloader/core/requesters/common_requester/__init__.py,sha256=kVKZyrS7PVlUnaV1xGsZdoW2J9XuyQ11A4oMV9Cc64Q,523
|
57
|
+
novel_downloader/core/requesters/common_requester/common_async_session.py,sha256=R67TcfYzVgKv7EMsNX2L1cviycfnR7qn0DIQ81ukK14,3538
|
52
58
|
novel_downloader/core/requesters/common_requester/common_session.py,sha256=EDl7yzO7tGf50xUqRSyA0g15XWNvVMgtdMDSGSOX9Gw,4650
|
53
59
|
novel_downloader/core/requesters/qidian_requester/__init__.py,sha256=s0ldqNvfqUsEnm_biM_bXEGN7gz88Z5IAx1OBvGW1lY,682
|
54
60
|
novel_downloader/core/requesters/qidian_requester/qidian_broswer.py,sha256=OqBp5gQ6eU3MuYuFP_lZeICG5f9J0DspAeHhYfmNGHw,14105
|
55
|
-
novel_downloader/core/requesters/qidian_requester/qidian_session.py,sha256=
|
61
|
+
novel_downloader/core/requesters/qidian_requester/qidian_session.py,sha256=cVq1G7IrQ8fHxIxEEUTX0txUMXuTVKkUsrDBPk52ngU,7664
|
56
62
|
novel_downloader/core/savers/__init__.py,sha256=p9O6p8ZUblrSheDVJoTSuDr6s1mJpQi8mz3QmQ16nHs,391
|
57
63
|
novel_downloader/core/savers/base_saver.py,sha256=VocVl8go80IkzAp9qY4dgZjmLbK8TVkg48Ugl53pxrc,5513
|
58
64
|
novel_downloader/core/savers/qidian_saver.py,sha256=MVAcWdM-IX_qsRW5It2aIkx9QPdRCLcZGcD3ihfm3gU,627
|
@@ -68,7 +74,7 @@ novel_downloader/core/savers/epub_utils/volume_intro.py,sha256=1NhnLKRL_ieoDgXTR
|
|
68
74
|
novel_downloader/locales/en.json,sha256=7xPlFLf6ByH0VMnGTTRC_6gRSW2IdTvQnKa5_FquSsk,5277
|
69
75
|
novel_downloader/locales/zh.json,sha256=TylYUKSUUbG4Fh_DQazUNTY96HriQWyBfKjh1FrI0xM,5163
|
70
76
|
novel_downloader/resources/config/rules.toml,sha256=hrED6h3Z3cjSY5hRPQhp4TFAU5QXnN9xHfVABOJQNrM,4979
|
71
|
-
novel_downloader/resources/config/settings.yaml,sha256=
|
77
|
+
novel_downloader/resources/config/settings.yaml,sha256=QSOhbK6orCJ5H9b9NZMIycMA546oJT2YB_bKN-Q5oI0,3364
|
72
78
|
novel_downloader/resources/css_styles/main.css,sha256=WM6GePwdOGgM86fbbOxQ0_0oerTBDZeQHt8zRVfcJp8,1617
|
73
79
|
novel_downloader/resources/css_styles/volume-intro.css,sha256=6gaUnNKkrb2w8tYJRq1BGD1FwbhT1I5W2GI_Zelo9G4,1156
|
74
80
|
novel_downloader/resources/images/volume_border.png,sha256=2dEVimnTHKOfLMhi7bhkh_5joWNnrqg8duomLSNOZx4,28613
|
@@ -78,7 +84,7 @@ novel_downloader/resources/text/blacklist.txt,sha256=sovK9JgARZP3lud5b1EZgvv8LSV
|
|
78
84
|
novel_downloader/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
79
85
|
novel_downloader/utils/cache.py,sha256=NB5j7CWNscfE4eFA0A9O5mYR1eW-216M-ljlMo0LDqE,615
|
80
86
|
novel_downloader/utils/constants.py,sha256=VETwA_prTjLUejmJgrRnOn0QbwNKO4aTHf41dJJy7uE,5505
|
81
|
-
novel_downloader/utils/crypto_utils.py,sha256=
|
87
|
+
novel_downloader/utils/crypto_utils.py,sha256=whGgir2oi_17pNteiIRztiMNaB-ZP63GMP3KPJkXA80,4178
|
82
88
|
novel_downloader/utils/hash_store.py,sha256=rpr61GsvZ9wT_1fEn4_83JZ-nWc1KLtcvb56ZqHawdk,9826
|
83
89
|
novel_downloader/utils/hash_utils.py,sha256=6S4-Q_uLNzdEDkBOUG9QEcflbuPFNDAYe6Gx718AOo4,2998
|
84
90
|
novel_downloader/utils/i18n.py,sha256=9vwBEuwk37ED29WI321-FWmA1_K4yv_qw84XUNbSr1s,1068
|
@@ -101,9 +107,9 @@ novel_downloader/utils/text_utils/text_cleaning.py,sha256=1yuaDeUBHqBRkkWhw43rV1
|
|
101
107
|
novel_downloader/utils/time_utils/__init__.py,sha256=bRpO14eorfH5C5xfqvW7QwSe3fQHhpr34j4O3qY5cGc,580
|
102
108
|
novel_downloader/utils/time_utils/datetime_utils.py,sha256=xYKuI2K6DKwZdfUBZ0j1SNbmHjhYU7hIu46NzlZqr3o,4887
|
103
109
|
novel_downloader/utils/time_utils/sleep_utils.py,sha256=CffWLotrhOZ-uYwC8Nb1cwZrAO2p83JDIrCGZLQuEC0,1384
|
104
|
-
novel_downloader-1.
|
105
|
-
novel_downloader-1.
|
106
|
-
novel_downloader-1.
|
107
|
-
novel_downloader-1.
|
108
|
-
novel_downloader-1.
|
109
|
-
novel_downloader-1.
|
110
|
+
novel_downloader-1.2.0.dist-info/licenses/LICENSE,sha256=XgmnH0mBf-qEiizoVAfJQAKzPB9y3rBa-ni7M0Aqv4A,1066
|
111
|
+
novel_downloader-1.2.0.dist-info/METADATA,sha256=kh4E7B8B4in3fwoP29FLqyVaSvEhVHNpJudtdp8WH4Q,6291
|
112
|
+
novel_downloader-1.2.0.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
|
113
|
+
novel_downloader-1.2.0.dist-info/entry_points.txt,sha256=v23QrJrfrAcYpxUYslCVxubOVRRTaTw7vlG_tfMsFP8,65
|
114
|
+
novel_downloader-1.2.0.dist-info/top_level.txt,sha256=hP4jYWM2LTm1jxsW4hqEB8N0dsRvldO2QdhggJT917I,17
|
115
|
+
novel_downloader-1.2.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|