jmcomic 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jmcomic/__init__.py +29 -0
- jmcomic/api.py +131 -0
- jmcomic/cl.py +121 -0
- jmcomic/jm_client_impl.py +1217 -0
- jmcomic/jm_client_interface.py +609 -0
- jmcomic/jm_config.py +505 -0
- jmcomic/jm_downloader.py +350 -0
- jmcomic/jm_entity.py +695 -0
- jmcomic/jm_exception.py +191 -0
- jmcomic/jm_option.py +647 -0
- jmcomic/jm_plugin.py +1203 -0
- jmcomic/jm_toolkit.py +937 -0
- jmcomic-0.0.2.dist-info/METADATA +229 -0
- jmcomic-0.0.2.dist-info/RECORD +18 -0
- jmcomic-0.0.2.dist-info/WHEEL +5 -0
- jmcomic-0.0.2.dist-info/entry_points.txt +2 -0
- jmcomic-0.0.2.dist-info/licenses/LICENSE +21 -0
- jmcomic-0.0.2.dist-info/top_level.txt +1 -0
jmcomic/jm_plugin.py
ADDED
|
@@ -0,0 +1,1203 @@
|
|
|
1
|
+
"""
|
|
2
|
+
该文件存放的是option插件
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .jm_option import *
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class PluginValidationException(Exception):
|
|
9
|
+
|
|
10
|
+
def __init__(self, plugin: 'JmOptionPlugin', msg: str):
|
|
11
|
+
self.plugin = plugin
|
|
12
|
+
self.msg = msg
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class JmOptionPlugin:
|
|
16
|
+
plugin_key: str
|
|
17
|
+
|
|
18
|
+
def __init__(self, option: JmOption):
|
|
19
|
+
self.option = option
|
|
20
|
+
self.log_enable = True
|
|
21
|
+
self.delete_original_file = False
|
|
22
|
+
|
|
23
|
+
def invoke(self, **kwargs) -> None:
|
|
24
|
+
"""
|
|
25
|
+
执行插件的功能
|
|
26
|
+
:param kwargs: 给插件的参数
|
|
27
|
+
"""
|
|
28
|
+
raise NotImplementedError
|
|
29
|
+
|
|
30
|
+
@classmethod
|
|
31
|
+
def build(cls, option: JmOption) -> 'JmOptionPlugin':
|
|
32
|
+
"""
|
|
33
|
+
创建插件实例
|
|
34
|
+
:param option: JmOption对象
|
|
35
|
+
"""
|
|
36
|
+
return cls(option)
|
|
37
|
+
|
|
38
|
+
def log(self, msg, topic=None):
|
|
39
|
+
if self.log_enable is not True:
|
|
40
|
+
return
|
|
41
|
+
|
|
42
|
+
jm_log(
|
|
43
|
+
topic=f'plugin.{self.plugin_key}' + (f'.{topic}' if topic is not None else ''),
|
|
44
|
+
msg=msg
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def require_param(self, case: Any, msg: str):
|
|
48
|
+
"""
|
|
49
|
+
专门用于校验参数的方法,会抛出特定异常,由option拦截根据策略进行处理
|
|
50
|
+
|
|
51
|
+
:param case: 条件
|
|
52
|
+
:param msg: 报错信息
|
|
53
|
+
"""
|
|
54
|
+
if case:
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
raise PluginValidationException(self, msg)
|
|
58
|
+
|
|
59
|
+
def warning_lib_not_install(self, lib: str, throw=False):
|
|
60
|
+
msg = (f'插件`{self.plugin_key}`依赖库: {lib},请先安装{lib}再使用。'
|
|
61
|
+
f'安装命令: [pip install {lib}]')
|
|
62
|
+
import warnings
|
|
63
|
+
warnings.warn(msg)
|
|
64
|
+
self.require_param(throw, msg)
|
|
65
|
+
|
|
66
|
+
def execute_deletion(self, paths: List[str]):
|
|
67
|
+
"""
|
|
68
|
+
删除文件和文件夹
|
|
69
|
+
:param paths: 路径列表
|
|
70
|
+
"""
|
|
71
|
+
if self.delete_original_file is not True:
|
|
72
|
+
return
|
|
73
|
+
|
|
74
|
+
for p in paths:
|
|
75
|
+
if file_not_exists(p):
|
|
76
|
+
continue
|
|
77
|
+
|
|
78
|
+
if os.path.isdir(p):
|
|
79
|
+
if os.listdir(p):
|
|
80
|
+
self.log(f'文件夹中存在非本次下载的文件,请手动删除文件夹内的文件: {p}', 'remove.ignore')
|
|
81
|
+
continue
|
|
82
|
+
os.rmdir(p)
|
|
83
|
+
self.log(f'删除文件夹: {p}', 'remove')
|
|
84
|
+
else:
|
|
85
|
+
os.remove(p)
|
|
86
|
+
self.log(f'删除原文件: {p}', 'remove')
|
|
87
|
+
|
|
88
|
+
# noinspection PyMethodMayBeStatic
|
|
89
|
+
def execute_cmd(self, cmd):
|
|
90
|
+
"""
|
|
91
|
+
执行shell命令,这里采用简单的实现
|
|
92
|
+
:param cmd: shell命令
|
|
93
|
+
"""
|
|
94
|
+
return os.system(cmd)
|
|
95
|
+
|
|
96
|
+
# noinspection PyMethodMayBeStatic
|
|
97
|
+
def execute_multi_line_cmd(self, cmd: str):
|
|
98
|
+
import subprocess
|
|
99
|
+
subprocess.run(cmd, shell=True, check=True)
|
|
100
|
+
|
|
101
|
+
def enter_wait_list(self):
|
|
102
|
+
self.option.need_wait_plugins.append(self)
|
|
103
|
+
|
|
104
|
+
def leave_wait_list(self):
|
|
105
|
+
self.option.need_wait_plugins.remove(self)
|
|
106
|
+
|
|
107
|
+
def wait_until_finish(self):
|
|
108
|
+
pass
|
|
109
|
+
|
|
110
|
+
# noinspection PyMethodMayBeStatic
|
|
111
|
+
def decide_filepath(self,
|
|
112
|
+
album: Optional[JmAlbumDetail],
|
|
113
|
+
photo: Optional[JmPhotoDetail],
|
|
114
|
+
filename_rule: str, suffix: str, base_dir: Optional[str],
|
|
115
|
+
dir_rule_dict: Optional[dict]
|
|
116
|
+
):
|
|
117
|
+
"""
|
|
118
|
+
根据规则计算一个文件的全路径
|
|
119
|
+
|
|
120
|
+
参数 dir_rule_dict 优先级最高,
|
|
121
|
+
如果 dir_rule_dict 不为空,优先用 dir_rule_dict
|
|
122
|
+
否则使用 base_dir + filename_rule + suffix
|
|
123
|
+
"""
|
|
124
|
+
filepath: str
|
|
125
|
+
base_dir: str
|
|
126
|
+
if dir_rule_dict is not None:
|
|
127
|
+
dir_rule = DirRule(**dir_rule_dict)
|
|
128
|
+
filepath = dir_rule.apply_rule_to_path(album, photo)
|
|
129
|
+
base_dir = os.path.dirname(filepath)
|
|
130
|
+
else:
|
|
131
|
+
base_dir = base_dir or os.getcwd()
|
|
132
|
+
filepath = os.path.join(base_dir, DirRule.apply_rule_to_filename(album, photo, filename_rule) + fix_suffix(suffix))
|
|
133
|
+
|
|
134
|
+
mkdir_if_not_exists(base_dir)
|
|
135
|
+
return filepath
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class JmLoginPlugin(JmOptionPlugin):
|
|
139
|
+
"""
|
|
140
|
+
功能:登录禁漫,并保存登录后的cookies,让所有client都带上此cookies
|
|
141
|
+
"""
|
|
142
|
+
plugin_key = 'login'
|
|
143
|
+
|
|
144
|
+
def invoke(self,
|
|
145
|
+
username: str,
|
|
146
|
+
password: str,
|
|
147
|
+
impl=None,
|
|
148
|
+
) -> None:
|
|
149
|
+
self.require_param(username, '用户名不能为空')
|
|
150
|
+
self.require_param(password, '密码不能为空')
|
|
151
|
+
|
|
152
|
+
client = self.option.build_jm_client(impl=impl)
|
|
153
|
+
client.login(username, password)
|
|
154
|
+
|
|
155
|
+
cookies = dict(client['cookies'])
|
|
156
|
+
self.option.update_cookies(cookies)
|
|
157
|
+
JmModuleConfig.APP_COOKIES = cookies
|
|
158
|
+
|
|
159
|
+
self.log('登录成功')
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class UsageLogPlugin(JmOptionPlugin):
|
|
163
|
+
plugin_key = 'usage_log'
|
|
164
|
+
|
|
165
|
+
def invoke(self, **kwargs) -> None:
|
|
166
|
+
import threading
|
|
167
|
+
t = threading.Thread(
|
|
168
|
+
target=self.monitor_resource_usage,
|
|
169
|
+
kwargs=kwargs,
|
|
170
|
+
daemon=True,
|
|
171
|
+
)
|
|
172
|
+
t.start()
|
|
173
|
+
|
|
174
|
+
self.set_thread_as_option_attr(t)
|
|
175
|
+
|
|
176
|
+
def set_thread_as_option_attr(self, t):
|
|
177
|
+
"""
|
|
178
|
+
线程留痕
|
|
179
|
+
"""
|
|
180
|
+
name = f'thread_{self.plugin_key}'
|
|
181
|
+
|
|
182
|
+
thread_ls: Optional[list] = getattr(self.option, name, None)
|
|
183
|
+
if thread_ls is None:
|
|
184
|
+
setattr(self.option, name, [t])
|
|
185
|
+
else:
|
|
186
|
+
thread_ls.append(t)
|
|
187
|
+
|
|
188
|
+
def monitor_resource_usage(
|
|
189
|
+
self,
|
|
190
|
+
interval=1,
|
|
191
|
+
enable_warning=True,
|
|
192
|
+
warning_cpu_percent=70,
|
|
193
|
+
warning_mem_percent=70,
|
|
194
|
+
warning_thread_count=100,
|
|
195
|
+
):
|
|
196
|
+
try:
|
|
197
|
+
import psutil
|
|
198
|
+
except ImportError:
|
|
199
|
+
self.warning_lib_not_install('psutil')
|
|
200
|
+
return
|
|
201
|
+
|
|
202
|
+
from time import sleep
|
|
203
|
+
from threading import active_count
|
|
204
|
+
# 获取当前进程
|
|
205
|
+
process = psutil.Process()
|
|
206
|
+
|
|
207
|
+
cpu_percent = None
|
|
208
|
+
# noinspection PyUnusedLocal
|
|
209
|
+
thread_count = None
|
|
210
|
+
# noinspection PyUnusedLocal
|
|
211
|
+
mem_usage = None
|
|
212
|
+
|
|
213
|
+
def warning():
|
|
214
|
+
warning_msg_list = []
|
|
215
|
+
if cpu_percent >= warning_cpu_percent:
|
|
216
|
+
warning_msg_list.append(f'进程占用cpu过高 ({cpu_percent}% >= {warning_cpu_percent}%)')
|
|
217
|
+
|
|
218
|
+
mem_percent = psutil.virtual_memory().percent
|
|
219
|
+
if mem_percent >= warning_mem_percent:
|
|
220
|
+
warning_msg_list.append(f'系统内存占用过高 ({mem_percent}% >= {warning_mem_percent}%)')
|
|
221
|
+
|
|
222
|
+
if thread_count >= warning_thread_count:
|
|
223
|
+
warning_msg_list.append(f'线程数过多 ({thread_count} >= {warning_thread_count})')
|
|
224
|
+
|
|
225
|
+
if len(warning_msg_list) != 0:
|
|
226
|
+
warning_msg_list.insert(0, '硬件占用告警,占用过高可能导致系统卡死!')
|
|
227
|
+
warning_msg_list.append('')
|
|
228
|
+
self.log('\n'.join(warning_msg_list), topic='warning')
|
|
229
|
+
|
|
230
|
+
while True:
|
|
231
|
+
# 获取CPU占用率(0~100)
|
|
232
|
+
cpu_percent = process.cpu_percent()
|
|
233
|
+
# 获取内存占用(MB)
|
|
234
|
+
mem_usage = round(process.memory_info().rss / 1024 / 1024, 2)
|
|
235
|
+
thread_count = active_count()
|
|
236
|
+
# 获取网络占用情况
|
|
237
|
+
# network_info = psutil.net_io_counters()
|
|
238
|
+
# network_bytes_sent = network_info.bytes_sent
|
|
239
|
+
# network_bytes_received = network_info.bytes_recv
|
|
240
|
+
|
|
241
|
+
# 打印信息
|
|
242
|
+
msg = ', '.join([
|
|
243
|
+
f'线程数: {thread_count}',
|
|
244
|
+
f'CPU占用: {cpu_percent}%',
|
|
245
|
+
f'内存占用: {mem_usage}MB',
|
|
246
|
+
# f"发送的字节数: {network_bytes_sent}",
|
|
247
|
+
# f"接收的字节数: {network_bytes_received}",
|
|
248
|
+
])
|
|
249
|
+
self.log(msg, topic='log')
|
|
250
|
+
|
|
251
|
+
if enable_warning is True:
|
|
252
|
+
# 警告
|
|
253
|
+
warning()
|
|
254
|
+
|
|
255
|
+
# 等待一段时间
|
|
256
|
+
sleep(interval)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class FindUpdatePlugin(JmOptionPlugin):
|
|
260
|
+
"""
|
|
261
|
+
参考: https://github.com/hect0x7/JMComic-Crawler-Python/issues/95
|
|
262
|
+
"""
|
|
263
|
+
plugin_key = 'find_update'
|
|
264
|
+
|
|
265
|
+
def invoke(self, **kwargs) -> None:
|
|
266
|
+
self.download_album_with_find_update(kwargs or {})
|
|
267
|
+
|
|
268
|
+
def download_album_with_find_update(self, dic: Dict[str, int]):
|
|
269
|
+
from .api import download_album
|
|
270
|
+
from .jm_downloader import JmDownloader
|
|
271
|
+
|
|
272
|
+
# 带入漫画id, 章节id(第x章),寻找该漫画下第x章节後的所有章节Id
|
|
273
|
+
def find_update(album: JmAlbumDetail):
|
|
274
|
+
if album.album_id not in dic:
|
|
275
|
+
return album
|
|
276
|
+
|
|
277
|
+
photo_ls = []
|
|
278
|
+
photo_begin = int(dic[album.album_id])
|
|
279
|
+
is_new_photo = False
|
|
280
|
+
|
|
281
|
+
for photo in album:
|
|
282
|
+
if is_new_photo:
|
|
283
|
+
photo_ls.append(photo)
|
|
284
|
+
|
|
285
|
+
if int(photo.photo_id) == photo_begin:
|
|
286
|
+
is_new_photo = True
|
|
287
|
+
|
|
288
|
+
return photo_ls
|
|
289
|
+
|
|
290
|
+
class FindUpdateDownloader(JmDownloader):
|
|
291
|
+
def do_filter(self, detail):
|
|
292
|
+
if not detail.is_album():
|
|
293
|
+
return detail
|
|
294
|
+
|
|
295
|
+
detail: JmAlbumDetail
|
|
296
|
+
return find_update(detail)
|
|
297
|
+
|
|
298
|
+
# 调用下载api,指定option和downloader
|
|
299
|
+
download_album(
|
|
300
|
+
jm_album_id=dic.keys(),
|
|
301
|
+
option=self.option,
|
|
302
|
+
downloader=FindUpdateDownloader,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
class ZipPlugin(JmOptionPlugin):
|
|
307
|
+
"""
|
|
308
|
+
感谢zip加密功能的贡献者:
|
|
309
|
+
- AXIS5 a.k.a AXIS5Hacker (https://github.com/hect0x7/JMComic-Crawler-Python/pull/375)
|
|
310
|
+
"""
|
|
311
|
+
|
|
312
|
+
plugin_key = 'zip'
|
|
313
|
+
|
|
314
|
+
# noinspection PyAttributeOutsideInit
|
|
315
|
+
def invoke(self,
|
|
316
|
+
downloader,
|
|
317
|
+
album: JmAlbumDetail = None,
|
|
318
|
+
photo: JmPhotoDetail = None,
|
|
319
|
+
delete_original_file=False,
|
|
320
|
+
level='photo',
|
|
321
|
+
filename_rule='Ptitle',
|
|
322
|
+
suffix='zip',
|
|
323
|
+
zip_dir='./',
|
|
324
|
+
dir_rule=None,
|
|
325
|
+
encrypt=None,
|
|
326
|
+
) -> None:
|
|
327
|
+
|
|
328
|
+
from .jm_downloader import JmDownloader
|
|
329
|
+
downloader: JmDownloader
|
|
330
|
+
self.downloader = downloader
|
|
331
|
+
self.level = level
|
|
332
|
+
self.delete_original_file = delete_original_file
|
|
333
|
+
|
|
334
|
+
# 确保压缩文件所在文件夹存在
|
|
335
|
+
zip_dir = JmcomicText.parse_to_abspath(zip_dir)
|
|
336
|
+
mkdir_if_not_exists(zip_dir)
|
|
337
|
+
|
|
338
|
+
path_to_delete = []
|
|
339
|
+
photo_dict = self.get_downloaded_photo(downloader, album, photo)
|
|
340
|
+
|
|
341
|
+
if level == 'album':
|
|
342
|
+
zip_path = self.decide_filepath(album, None, filename_rule, suffix, zip_dir, dir_rule)
|
|
343
|
+
self.zip_album(album, photo_dict, zip_path, path_to_delete, encrypt)
|
|
344
|
+
|
|
345
|
+
elif level == 'photo':
|
|
346
|
+
for photo, image_list in photo_dict.items():
|
|
347
|
+
zip_path = self.decide_filepath(photo.from_album, photo, filename_rule, suffix, zip_dir, dir_rule)
|
|
348
|
+
self.zip_photo(photo, image_list, zip_path, path_to_delete, encrypt)
|
|
349
|
+
|
|
350
|
+
else:
|
|
351
|
+
ExceptionTool.raises(f'Not Implemented Zip Level: {level}')
|
|
352
|
+
|
|
353
|
+
self.after_zip(path_to_delete)
|
|
354
|
+
|
|
355
|
+
# noinspection PyMethodMayBeStatic
|
|
356
|
+
def get_downloaded_photo(self, downloader, album, photo):
|
|
357
|
+
return (
|
|
358
|
+
downloader.download_success_dict[album]
|
|
359
|
+
if album is not None # after_album
|
|
360
|
+
else downloader.download_success_dict[photo.from_album] # after_photo
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
def zip_photo(self, photo, image_list: list, zip_path: str, path_to_delete, encrypt_dict):
|
|
364
|
+
"""
|
|
365
|
+
压缩photo文件夹
|
|
366
|
+
"""
|
|
367
|
+
photo_dir = self.option.decide_image_save_dir(photo) \
|
|
368
|
+
if len(image_list) == 0 \
|
|
369
|
+
else os.path.dirname(image_list[0][0])
|
|
370
|
+
|
|
371
|
+
with self.open_zip_file(zip_path, encrypt_dict) as f:
|
|
372
|
+
for file in files_of_dir(photo_dir):
|
|
373
|
+
abspath = os.path.join(photo_dir, file)
|
|
374
|
+
relpath = os.path.relpath(abspath, photo_dir)
|
|
375
|
+
f.write(abspath, relpath)
|
|
376
|
+
|
|
377
|
+
self.log(f'压缩章节[{photo.photo_id}]成功 → {zip_path}', 'finish')
|
|
378
|
+
path_to_delete.append(self.unified_path(photo_dir))
|
|
379
|
+
|
|
380
|
+
@staticmethod
|
|
381
|
+
def unified_path(f):
|
|
382
|
+
return fix_filepath(f, os.path.isdir(f))
|
|
383
|
+
|
|
384
|
+
def zip_album(self, album, photo_dict: dict, zip_path, path_to_delete, encrypt_dict):
|
|
385
|
+
"""
|
|
386
|
+
压缩album文件夹
|
|
387
|
+
"""
|
|
388
|
+
|
|
389
|
+
album_dir = self.option.dir_rule.decide_album_root_dir(album)
|
|
390
|
+
with self.open_zip_file(zip_path, encrypt_dict) as f:
|
|
391
|
+
for photo in photo_dict.keys():
|
|
392
|
+
# 定位到章节所在文件夹
|
|
393
|
+
photo_dir = self.unified_path(self.option.decide_image_save_dir(photo))
|
|
394
|
+
# 章节文件夹标记为删除
|
|
395
|
+
path_to_delete.append(photo_dir)
|
|
396
|
+
for file in files_of_dir(photo_dir):
|
|
397
|
+
abspath = os.path.join(photo_dir, file)
|
|
398
|
+
relpath = os.path.relpath(abspath, album_dir)
|
|
399
|
+
f.write(abspath, relpath)
|
|
400
|
+
self.log(f'压缩本子[{album.album_id}]成功 → {zip_path}', 'finish')
|
|
401
|
+
|
|
402
|
+
def after_zip(self, path_to_delete: List[str]):
|
|
403
|
+
# 删除所有原文件
|
|
404
|
+
dirs = sorted(path_to_delete, reverse=True)
|
|
405
|
+
image_paths = [
|
|
406
|
+
path
|
|
407
|
+
for photo_dict in self.downloader.download_success_dict.values()
|
|
408
|
+
for image_list in photo_dict.values()
|
|
409
|
+
for path, image in image_list
|
|
410
|
+
]
|
|
411
|
+
self.execute_deletion(image_paths)
|
|
412
|
+
self.execute_deletion(dirs)
|
|
413
|
+
|
|
414
|
+
# noinspection PyMethodMayBeStatic
|
|
415
|
+
@classmethod
|
|
416
|
+
def generate_random_str(cls, random_length) -> str:
|
|
417
|
+
"""
|
|
418
|
+
自动生成随机字符密码,长度由randomlength指定
|
|
419
|
+
"""
|
|
420
|
+
import random
|
|
421
|
+
|
|
422
|
+
random_str = ''
|
|
423
|
+
base_str = r'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='
|
|
424
|
+
base_length = len(base_str) - 1
|
|
425
|
+
for _ in range(random_length):
|
|
426
|
+
random_str += base_str[random.randint(0, base_length)]
|
|
427
|
+
return random_str
|
|
428
|
+
|
|
429
|
+
def open_zip_file(self, zip_path: str, encrypt_dict: Optional[dict]):
|
|
430
|
+
if encrypt_dict is None:
|
|
431
|
+
import zipfile
|
|
432
|
+
return zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
|
|
433
|
+
|
|
434
|
+
password, is_random = self.decide_password(encrypt_dict, zip_path)
|
|
435
|
+
if encrypt_dict.get('impl', '') == '7z':
|
|
436
|
+
try:
|
|
437
|
+
# noinspection PyUnresolvedReferences
|
|
438
|
+
import py7zr
|
|
439
|
+
except ImportError:
|
|
440
|
+
self.warning_lib_not_install('py7zr', True)
|
|
441
|
+
|
|
442
|
+
# noinspection PyUnboundLocalVariable
|
|
443
|
+
filters = [{'id': py7zr.FILTER_COPY}]
|
|
444
|
+
return py7zr.SevenZipFile(zip_path, mode='w', password=password, filters=filters, header_encryption=True)
|
|
445
|
+
else:
|
|
446
|
+
try:
|
|
447
|
+
# noinspection PyUnresolvedReferences
|
|
448
|
+
import pyzipper
|
|
449
|
+
except ImportError:
|
|
450
|
+
self.warning_lib_not_install('pyzipper', True)
|
|
451
|
+
|
|
452
|
+
# noinspection PyUnboundLocalVariable
|
|
453
|
+
aes_zip_file = pyzipper.AESZipFile(zip_path, "w", pyzipper.ZIP_DEFLATED)
|
|
454
|
+
aes_zip_file.setencryption(pyzipper.WZ_AES, nbits=128)
|
|
455
|
+
password_bytes = str.encode(password)
|
|
456
|
+
aes_zip_file.setpassword(password_bytes)
|
|
457
|
+
if is_random:
|
|
458
|
+
aes_zip_file.comment = password_bytes
|
|
459
|
+
return aes_zip_file
|
|
460
|
+
|
|
461
|
+
def decide_password(self, encrypt_dict: dict, zip_path: str):
|
|
462
|
+
encrypt_type = encrypt_dict.get('type', '')
|
|
463
|
+
is_random = False
|
|
464
|
+
|
|
465
|
+
if encrypt_type == 'random':
|
|
466
|
+
is_random = True
|
|
467
|
+
password = self.generate_random_str(48)
|
|
468
|
+
self.log(f'生成随机密码: [{password}] → [{zip_path}]', 'encrypt')
|
|
469
|
+
else:
|
|
470
|
+
password = str(encrypt_dict['password'])
|
|
471
|
+
self.log(f'使用指定密码: [{password}] → [{zip_path}]', 'encrypt')
|
|
472
|
+
|
|
473
|
+
return password, is_random
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
class ClientProxyPlugin(JmOptionPlugin):
|
|
477
|
+
plugin_key = 'client_proxy'
|
|
478
|
+
|
|
479
|
+
def invoke(self,
|
|
480
|
+
proxy_client_key,
|
|
481
|
+
whitelist=None,
|
|
482
|
+
**clazz_init_kwargs,
|
|
483
|
+
) -> None:
|
|
484
|
+
if whitelist is not None:
|
|
485
|
+
whitelist = set(whitelist)
|
|
486
|
+
|
|
487
|
+
proxy_clazz = JmModuleConfig.client_impl_class(proxy_client_key)
|
|
488
|
+
new_jm_client: Callable = self.option.new_jm_client
|
|
489
|
+
|
|
490
|
+
def hook_new_jm_client(*args, **kwargs):
|
|
491
|
+
client = new_jm_client(*args, **kwargs)
|
|
492
|
+
if whitelist is not None and client.client_key not in whitelist:
|
|
493
|
+
return client
|
|
494
|
+
|
|
495
|
+
self.log(f'proxy client {client} with {proxy_clazz}')
|
|
496
|
+
return proxy_clazz(client, **clazz_init_kwargs)
|
|
497
|
+
|
|
498
|
+
self.option.new_jm_client = hook_new_jm_client
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
class ImageSuffixFilterPlugin(JmOptionPlugin):
|
|
502
|
+
plugin_key = 'image_suffix_filter'
|
|
503
|
+
|
|
504
|
+
def invoke(self,
|
|
505
|
+
allowed_orig_suffix=None,
|
|
506
|
+
) -> None:
|
|
507
|
+
if allowed_orig_suffix is None:
|
|
508
|
+
return
|
|
509
|
+
|
|
510
|
+
allowed_suffix_set = set(fix_suffix(suffix) for suffix in allowed_orig_suffix)
|
|
511
|
+
|
|
512
|
+
option_decide_cache = self.option.decide_download_cache
|
|
513
|
+
|
|
514
|
+
def apply_filter_then_decide_cache(image: JmImageDetail):
|
|
515
|
+
if image.img_file_suffix not in allowed_suffix_set:
|
|
516
|
+
self.log(f'跳过下载图片: {image.tag},'
|
|
517
|
+
f'因为其后缀\'{image.img_file_suffix}\'不在允许的后缀集合{allowed_suffix_set}内')
|
|
518
|
+
image.skip = True
|
|
519
|
+
|
|
520
|
+
# let option decide
|
|
521
|
+
return option_decide_cache(image)
|
|
522
|
+
|
|
523
|
+
self.option.decide_download_cache = apply_filter_then_decide_cache
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
class SendQQEmailPlugin(JmOptionPlugin):
|
|
527
|
+
plugin_key = 'send_qq_email'
|
|
528
|
+
|
|
529
|
+
def invoke(self,
|
|
530
|
+
msg_from,
|
|
531
|
+
msg_to,
|
|
532
|
+
password,
|
|
533
|
+
title,
|
|
534
|
+
content,
|
|
535
|
+
album=None,
|
|
536
|
+
downloader=None,
|
|
537
|
+
) -> None:
|
|
538
|
+
self.require_param(msg_from and msg_to and password, '发件人、收件人、授权码都不能为空')
|
|
539
|
+
|
|
540
|
+
from common import EmailConfig
|
|
541
|
+
econfig = EmailConfig(msg_from, msg_to, password)
|
|
542
|
+
epostman = econfig.create_email_postman()
|
|
543
|
+
epostman.send(content, title)
|
|
544
|
+
|
|
545
|
+
self.log('Email sent successfully')
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
class LogTopicFilterPlugin(JmOptionPlugin):
|
|
549
|
+
plugin_key = 'log_topic_filter'
|
|
550
|
+
|
|
551
|
+
def invoke(self, whitelist) -> None:
|
|
552
|
+
if whitelist is not None:
|
|
553
|
+
whitelist = set(whitelist)
|
|
554
|
+
|
|
555
|
+
old_jm_log = JmModuleConfig.EXECUTOR_LOG
|
|
556
|
+
|
|
557
|
+
def new_jm_log(topic, msg):
|
|
558
|
+
if whitelist is not None and topic not in whitelist:
|
|
559
|
+
return
|
|
560
|
+
|
|
561
|
+
old_jm_log(topic, msg)
|
|
562
|
+
|
|
563
|
+
JmModuleConfig.EXECUTOR_LOG = new_jm_log
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
class AutoSetBrowserCookiesPlugin(JmOptionPlugin):
|
|
567
|
+
plugin_key = 'auto_set_browser_cookies'
|
|
568
|
+
|
|
569
|
+
accepted_cookies_keys = str_to_set('''
|
|
570
|
+
yuo1
|
|
571
|
+
remember_id
|
|
572
|
+
remember
|
|
573
|
+
''')
|
|
574
|
+
|
|
575
|
+
def invoke(self,
|
|
576
|
+
browser: str,
|
|
577
|
+
domain: str,
|
|
578
|
+
) -> None:
|
|
579
|
+
"""
|
|
580
|
+
坑点预警:由于禁漫需要校验同一设备,使用该插件需要配置自己浏览器的headers,例如
|
|
581
|
+
|
|
582
|
+
```yml
|
|
583
|
+
client:
|
|
584
|
+
postman:
|
|
585
|
+
meta_data:
|
|
586
|
+
headers: {
|
|
587
|
+
# 浏览器headers
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
# 插件配置如下:
|
|
591
|
+
plugins:
|
|
592
|
+
after_init:
|
|
593
|
+
- plugin: auto_set_browser_cookies
|
|
594
|
+
kwargs:
|
|
595
|
+
browser: chrome
|
|
596
|
+
domain: 18comic.vip
|
|
597
|
+
```
|
|
598
|
+
|
|
599
|
+
:param browser: chrome/edge/...
|
|
600
|
+
:param domain: 18comic.vip/...
|
|
601
|
+
:return: cookies
|
|
602
|
+
"""
|
|
603
|
+
cookies, e = get_browser_cookies(browser, domain, safe=True)
|
|
604
|
+
|
|
605
|
+
if cookies is None:
|
|
606
|
+
if isinstance(e, ImportError):
|
|
607
|
+
self.warning_lib_not_install('browser_cookie3')
|
|
608
|
+
else:
|
|
609
|
+
self.log('获取浏览器cookies失败,请关闭浏览器重试')
|
|
610
|
+
return
|
|
611
|
+
|
|
612
|
+
self.option.update_cookies(
|
|
613
|
+
{k: v for k, v in cookies.items() if k in self.accepted_cookies_keys}
|
|
614
|
+
)
|
|
615
|
+
self.log('获取浏览器cookies成功')
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
# noinspection PyMethodMayBeStatic
|
|
619
|
+
class FavoriteFolderExportPlugin(JmOptionPlugin):
|
|
620
|
+
plugin_key = 'favorite_folder_export'
|
|
621
|
+
|
|
622
|
+
# noinspection PyAttributeOutsideInit
|
|
623
|
+
def invoke(self,
|
|
624
|
+
save_dir=None,
|
|
625
|
+
zip_enable=False,
|
|
626
|
+
zip_filepath=None,
|
|
627
|
+
zip_password=None,
|
|
628
|
+
delete_original_file=False,
|
|
629
|
+
):
|
|
630
|
+
self.save_dir = os.path.abspath(save_dir if save_dir is not None else (os.getcwd() + '/export/'))
|
|
631
|
+
self.zip_enable = zip_enable
|
|
632
|
+
self.zip_filepath = os.path.abspath(zip_filepath)
|
|
633
|
+
self.zip_password = zip_password
|
|
634
|
+
self.delete_original_file = delete_original_file
|
|
635
|
+
self.files = []
|
|
636
|
+
|
|
637
|
+
mkdir_if_not_exists(self.save_dir)
|
|
638
|
+
mkdir_if_not_exists(of_dir_path(self.zip_filepath))
|
|
639
|
+
|
|
640
|
+
self.main()
|
|
641
|
+
|
|
642
|
+
def main(self):
|
|
643
|
+
cl = self.option.build_jm_client()
|
|
644
|
+
# noinspection PyAttributeOutsideInit
|
|
645
|
+
self.cl = cl
|
|
646
|
+
page = cl.favorite_folder()
|
|
647
|
+
|
|
648
|
+
# 获取所有的收藏夹
|
|
649
|
+
folders = {fid: fname for fid, fname in page.iter_folder_id_name()}
|
|
650
|
+
# 加上特殊收藏栏【全部】
|
|
651
|
+
folders.setdefault('0', '全部')
|
|
652
|
+
|
|
653
|
+
# 一个收藏夹一个线程,导出收藏夹数据到文件
|
|
654
|
+
multi_thread_launcher(
|
|
655
|
+
iter_objs=folders.items(),
|
|
656
|
+
apply_each_obj_func=self.handle_folder,
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
if not self.zip_enable:
|
|
660
|
+
return
|
|
661
|
+
|
|
662
|
+
# 压缩导出的文件
|
|
663
|
+
self.require_param(self.zip_filepath, '如果开启zip,请指定zip_filepath参数(压缩文件保存路径)')
|
|
664
|
+
|
|
665
|
+
if self.zip_password is None:
|
|
666
|
+
self.zip_folder_without_password(self.files, self.zip_filepath)
|
|
667
|
+
else:
|
|
668
|
+
self.zip_with_password()
|
|
669
|
+
|
|
670
|
+
self.execute_deletion(self.files)
|
|
671
|
+
|
|
672
|
+
def handle_folder(self, fid: str, fname: str):
|
|
673
|
+
self.log(f'【收藏夹: {fname}, fid: {fid}】开始获取数据')
|
|
674
|
+
|
|
675
|
+
# 获取收藏夹数据
|
|
676
|
+
page_data = self.fetch_folder_page_data(fid)
|
|
677
|
+
|
|
678
|
+
# 序列化到文件
|
|
679
|
+
filepath = self.save_folder_page_data_to_file(page_data, fid, fname)
|
|
680
|
+
|
|
681
|
+
if filepath is None:
|
|
682
|
+
self.log(f'【收藏夹: {fname}, fid: {fid}】收藏夹无数据')
|
|
683
|
+
return
|
|
684
|
+
|
|
685
|
+
self.log(f'【收藏夹: {fname}, fid: {fid}】保存文件成功 → [{filepath}]')
|
|
686
|
+
self.files.append(filepath)
|
|
687
|
+
|
|
688
|
+
def fetch_folder_page_data(self, fid):
|
|
689
|
+
# 一页一页获取,不使用并行
|
|
690
|
+
page_data = list(self.cl.favorite_folder_gen(folder_id=fid))
|
|
691
|
+
return page_data
|
|
692
|
+
|
|
693
|
+
def save_folder_page_data_to_file(self, page_data: List[JmFavoritePage], fid: str, fname: str):
|
|
694
|
+
from os import path
|
|
695
|
+
filepath = path.abspath(path.join(self.save_dir, fix_windir_name(f'【{fid}】{fname}.csv')))
|
|
696
|
+
|
|
697
|
+
data = []
|
|
698
|
+
for page in page_data:
|
|
699
|
+
for aid, extra in page.content:
|
|
700
|
+
data.append(
|
|
701
|
+
(aid, extra.get('author', '') or JmModuleConfig.DEFAULT_AUTHOR, extra['name'])
|
|
702
|
+
)
|
|
703
|
+
|
|
704
|
+
if len(data) == 0:
|
|
705
|
+
return
|
|
706
|
+
|
|
707
|
+
with open(filepath, 'w', encoding='utf-8') as f:
|
|
708
|
+
f.write('id,author,name\n')
|
|
709
|
+
for item in data:
|
|
710
|
+
f.write(','.join(item) + '\n')
|
|
711
|
+
|
|
712
|
+
return filepath
|
|
713
|
+
|
|
714
|
+
def zip_folder_without_password(self, files, zip_path):
|
|
715
|
+
"""
|
|
716
|
+
压缩文件夹中的文件并设置密码
|
|
717
|
+
|
|
718
|
+
:param files: 要压缩的文件的绝对路径的列表
|
|
719
|
+
:param zip_path: 压缩文件的保存路径
|
|
720
|
+
"""
|
|
721
|
+
import zipfile
|
|
722
|
+
|
|
723
|
+
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
|
|
724
|
+
# 获取文件夹中的文件列表并将其添加到 ZIP 文件中
|
|
725
|
+
for file in files:
|
|
726
|
+
zipf.write(file, arcname=of_file_name(file))
|
|
727
|
+
|
|
728
|
+
def zip_with_password(self):
|
|
729
|
+
# 构造shell命令
|
|
730
|
+
cmd_list = f'''
|
|
731
|
+
cd {self.save_dir}
|
|
732
|
+
7z a "{self.zip_filepath}" "./" -p{self.zip_password} -mhe=on > "../7z_output.txt"
|
|
733
|
+
|
|
734
|
+
'''
|
|
735
|
+
self.log(f'运行命令: {cmd_list}')
|
|
736
|
+
|
|
737
|
+
# 执行
|
|
738
|
+
self.execute_multi_line_cmd(cmd_list)
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
class Img2pdfPlugin(JmOptionPlugin):
|
|
742
|
+
plugin_key = 'img2pdf'
|
|
743
|
+
|
|
744
|
+
def invoke(self,
|
|
745
|
+
photo: JmPhotoDetail = None,
|
|
746
|
+
album: JmAlbumDetail = None,
|
|
747
|
+
downloader=None,
|
|
748
|
+
pdf_dir=None,
|
|
749
|
+
filename_rule='Pid',
|
|
750
|
+
dir_rule=None,
|
|
751
|
+
delete_original_file=False,
|
|
752
|
+
**kwargs,
|
|
753
|
+
):
|
|
754
|
+
if photo is None and album is None:
|
|
755
|
+
jm_log('wrong_usage', 'img2pdf必须运行在after_photo或after_album时')
|
|
756
|
+
|
|
757
|
+
try:
|
|
758
|
+
import img2pdf
|
|
759
|
+
except ImportError:
|
|
760
|
+
self.warning_lib_not_install('img2pdf')
|
|
761
|
+
return
|
|
762
|
+
|
|
763
|
+
self.delete_original_file = delete_original_file
|
|
764
|
+
|
|
765
|
+
# 处理生成的pdf文件的路径
|
|
766
|
+
pdf_filepath = self.decide_filepath(album, photo, filename_rule, 'pdf', pdf_dir, dir_rule)
|
|
767
|
+
|
|
768
|
+
# 调用 img2pdf 把 photo_dir 下的所有图片转为pdf
|
|
769
|
+
img_path_ls, img_dir_ls = self.write_img_2_pdf(pdf_filepath, album, photo)
|
|
770
|
+
self.log(f'Convert Successfully: JM{album or photo} → {pdf_filepath}')
|
|
771
|
+
|
|
772
|
+
# 执行删除
|
|
773
|
+
img_path_ls += img_dir_ls
|
|
774
|
+
self.execute_deletion(img_path_ls)
|
|
775
|
+
|
|
776
|
+
def write_img_2_pdf(self, pdf_filepath, album: JmAlbumDetail, photo: JmPhotoDetail):
|
|
777
|
+
import img2pdf
|
|
778
|
+
|
|
779
|
+
if album is None:
|
|
780
|
+
img_dir_ls = [self.option.decide_image_save_dir(photo)]
|
|
781
|
+
else:
|
|
782
|
+
img_dir_ls = [self.option.decide_image_save_dir(photo) for photo in album]
|
|
783
|
+
|
|
784
|
+
img_path_ls = []
|
|
785
|
+
|
|
786
|
+
for img_dir in img_dir_ls:
|
|
787
|
+
imgs = files_of_dir(img_dir)
|
|
788
|
+
if not imgs:
|
|
789
|
+
continue
|
|
790
|
+
img_path_ls += imgs
|
|
791
|
+
|
|
792
|
+
if len(img_path_ls) == 0:
|
|
793
|
+
self.log(f'所有文件夹都不存在图片,无法生成pdf:{img_dir_ls}', 'error')
|
|
794
|
+
|
|
795
|
+
with open(pdf_filepath, 'wb') as f:
|
|
796
|
+
f.write(img2pdf.convert(img_path_ls))
|
|
797
|
+
|
|
798
|
+
return img_path_ls, img_dir_ls
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
class LongImgPlugin(JmOptionPlugin):
|
|
802
|
+
plugin_key = 'long_img'
|
|
803
|
+
|
|
804
|
+
def invoke(self,
|
|
805
|
+
photo: JmPhotoDetail = None,
|
|
806
|
+
album: JmAlbumDetail = None,
|
|
807
|
+
downloader=None,
|
|
808
|
+
img_dir=None,
|
|
809
|
+
filename_rule='Pid',
|
|
810
|
+
delete_original_file=False,
|
|
811
|
+
dir_rule=None,
|
|
812
|
+
**kwargs,
|
|
813
|
+
):
|
|
814
|
+
if photo is None and album is None:
|
|
815
|
+
jm_log('wrong_usage', 'long_img必须运行在after_photo或after_album时')
|
|
816
|
+
|
|
817
|
+
try:
|
|
818
|
+
from PIL import Image
|
|
819
|
+
except ImportError:
|
|
820
|
+
self.warning_lib_not_install('PIL')
|
|
821
|
+
return
|
|
822
|
+
|
|
823
|
+
self.delete_original_file = delete_original_file
|
|
824
|
+
|
|
825
|
+
# 处理生成的长图文件的路径
|
|
826
|
+
long_img_path = self.decide_filepath(album, photo, filename_rule, 'png', img_dir, dir_rule)
|
|
827
|
+
|
|
828
|
+
# 调用 PIL 把 photo_dir 下的所有图片合并为长图
|
|
829
|
+
img_path_ls = self.write_img_2_long_img(long_img_path, album, photo)
|
|
830
|
+
self.log(f'Convert Successfully: JM{album or photo} → {long_img_path}')
|
|
831
|
+
|
|
832
|
+
# 执行删除
|
|
833
|
+
self.execute_deletion(img_path_ls)
|
|
834
|
+
|
|
835
|
+
def write_img_2_long_img(self, long_img_path, album: JmAlbumDetail, photo: JmPhotoDetail) -> List[str]:
|
|
836
|
+
import itertools
|
|
837
|
+
from PIL import Image
|
|
838
|
+
|
|
839
|
+
if album is None:
|
|
840
|
+
img_dir_items = [self.option.decide_image_save_dir(photo)]
|
|
841
|
+
else:
|
|
842
|
+
img_dir_items = [self.option.decide_image_save_dir(photo) for photo in album]
|
|
843
|
+
|
|
844
|
+
img_paths = itertools.chain(*map(files_of_dir, img_dir_items))
|
|
845
|
+
img_paths = list(filter(lambda x: not x.startswith('.'), img_paths)) # 过滤系统文件
|
|
846
|
+
|
|
847
|
+
images = self.open_images(img_paths)
|
|
848
|
+
|
|
849
|
+
try:
|
|
850
|
+
resample_method = Image.Resampling.LANCZOS
|
|
851
|
+
except AttributeError:
|
|
852
|
+
resample_method = Image.LANCZOS
|
|
853
|
+
|
|
854
|
+
min_img_width = min(img.width for img in images)
|
|
855
|
+
total_height = 0
|
|
856
|
+
for i, img in enumerate(images):
|
|
857
|
+
if img.width > min_img_width:
|
|
858
|
+
images[i] = img.resize((min_img_width, int(img.height * min_img_width / img.width)),
|
|
859
|
+
resample=resample_method)
|
|
860
|
+
total_height += images[i].height
|
|
861
|
+
|
|
862
|
+
long_img = Image.new('RGB', (min_img_width, total_height))
|
|
863
|
+
y_offset = 0
|
|
864
|
+
for img in images:
|
|
865
|
+
long_img.paste(img, (0, y_offset))
|
|
866
|
+
y_offset += img.height
|
|
867
|
+
|
|
868
|
+
long_img.save(long_img_path)
|
|
869
|
+
for img in images:
|
|
870
|
+
img.close()
|
|
871
|
+
|
|
872
|
+
return img_paths
|
|
873
|
+
|
|
874
|
+
def open_images(self, img_paths: List[str]):
|
|
875
|
+
images = []
|
|
876
|
+
for img_path in img_paths:
|
|
877
|
+
try:
|
|
878
|
+
img = Image.open(img_path)
|
|
879
|
+
images.append(img)
|
|
880
|
+
except IOError as e:
|
|
881
|
+
self.log(f"Failed to open image {img_path}: {e}", 'error')
|
|
882
|
+
return images
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
class JmServerPlugin(JmOptionPlugin):
|
|
886
|
+
plugin_key = 'jm_server'
|
|
887
|
+
|
|
888
|
+
default_run_kwargs = {
|
|
889
|
+
'host': '0.0.0.0',
|
|
890
|
+
'port': '80',
|
|
891
|
+
'debug': False,
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
from threading import Lock
|
|
895
|
+
single_instance_lock = Lock()
|
|
896
|
+
|
|
897
|
+
def __init__(self, option: JmOption):
|
|
898
|
+
super().__init__(option)
|
|
899
|
+
self.run_server_lock = Lock()
|
|
900
|
+
self.running = False
|
|
901
|
+
self.server_thread: Optional[Thread] = None
|
|
902
|
+
|
|
903
|
+
def invoke(self,
|
|
904
|
+
password='',
|
|
905
|
+
base_dir=None,
|
|
906
|
+
album=None,
|
|
907
|
+
photo=None,
|
|
908
|
+
downloader=None,
|
|
909
|
+
run=None,
|
|
910
|
+
**kwargs
|
|
911
|
+
):
|
|
912
|
+
"""
|
|
913
|
+
|
|
914
|
+
:param password: 密码
|
|
915
|
+
:param base_dir: 初始访问服务器的根路径
|
|
916
|
+
:param album: 为了支持 after_album 这种调用时机
|
|
917
|
+
:param photo: 为了支持 after_album 这种调用时机
|
|
918
|
+
:param downloader: 为了支持 after_album 这种调用时机
|
|
919
|
+
:param run: 用于启动服务器: app.run(**run_kwargs)
|
|
920
|
+
:param kwargs: 用于JmServer构造函数: JmServer(base_dir, password, **kwargs)
|
|
921
|
+
"""
|
|
922
|
+
|
|
923
|
+
if base_dir is None:
|
|
924
|
+
base_dir = self.option.dir_rule.base_dir
|
|
925
|
+
|
|
926
|
+
if run is None:
|
|
927
|
+
run = self.default_run_kwargs
|
|
928
|
+
else:
|
|
929
|
+
base_run_kwargs = self.default_run_kwargs.copy()
|
|
930
|
+
base_run_kwargs.update(run)
|
|
931
|
+
run = base_run_kwargs
|
|
932
|
+
|
|
933
|
+
if self.running is True:
|
|
934
|
+
return
|
|
935
|
+
|
|
936
|
+
with self.run_server_lock:
|
|
937
|
+
if self.running is True:
|
|
938
|
+
return
|
|
939
|
+
|
|
940
|
+
# 服务器的代码位于一个独立库:plugin_jm_server,需要独立安装
|
|
941
|
+
# 源代码仓库:https://github.com/hect0x7/plugin-jm-server
|
|
942
|
+
try:
|
|
943
|
+
# noinspection PyUnresolvedReferences
|
|
944
|
+
import plugin_jm_server
|
|
945
|
+
self.log(f'当前使用plugin_jm_server版本: {plugin_jm_server.__version__}')
|
|
946
|
+
except ImportError:
|
|
947
|
+
self.warning_lib_not_install('plugin_jm_server')
|
|
948
|
+
return
|
|
949
|
+
|
|
950
|
+
# 核心函数,启动服务器,会阻塞当前线程
|
|
951
|
+
def blocking_run_server():
|
|
952
|
+
self.server_thread = current_thread()
|
|
953
|
+
self.enter_wait_list()
|
|
954
|
+
server = plugin_jm_server.JmServer(base_dir, password, **kwargs)
|
|
955
|
+
# run方法会阻塞当前线程直到flask退出
|
|
956
|
+
server.run(**run)
|
|
957
|
+
|
|
958
|
+
# 对于debug模式,特殊处理
|
|
959
|
+
if run['debug'] is True:
|
|
960
|
+
run.setdefault('use_reloader', False)
|
|
961
|
+
|
|
962
|
+
# debug模式只能在主线程启动,判断当前线程是不是主线程
|
|
963
|
+
if current_thread() is not threading.main_thread():
|
|
964
|
+
# 不是主线程,return
|
|
965
|
+
return self.warning_wrong_usage_of_debug()
|
|
966
|
+
else:
|
|
967
|
+
self.running = True
|
|
968
|
+
# 是主线程,启动服务器
|
|
969
|
+
blocking_run_server()
|
|
970
|
+
|
|
971
|
+
else:
|
|
972
|
+
# 非debug模式,开新线程启动
|
|
973
|
+
threading.Thread(target=blocking_run_server, daemon=True).start()
|
|
974
|
+
atexit_register(self.wait_server_stop)
|
|
975
|
+
self.running = True
|
|
976
|
+
|
|
977
|
+
def warning_wrong_usage_of_debug(self):
|
|
978
|
+
self.log('注意!当配置debug=True时,请确保当前插件是在主线程中被调用。\n'
|
|
979
|
+
'因为如果本插件配置在 [after_album/after_photo] 这种时机调用,\n'
|
|
980
|
+
'会使得flask框架不在主线程debug运行,\n'
|
|
981
|
+
'导致报错(ValueError: signal only works in main thread of the main interpreter)。\n',
|
|
982
|
+
'【基于上述原因,当前线程非主线程,不启动服务器】'
|
|
983
|
+
'warning'
|
|
984
|
+
)
|
|
985
|
+
|
|
986
|
+
def wait_server_stop(self, proactive=False):
|
|
987
|
+
st = self.server_thread
|
|
988
|
+
if (
|
|
989
|
+
st is None
|
|
990
|
+
or st == current_thread()
|
|
991
|
+
or not st.is_alive()
|
|
992
|
+
):
|
|
993
|
+
return
|
|
994
|
+
|
|
995
|
+
if proactive:
|
|
996
|
+
msg = f'[{self.plugin_key}]的服务器线程仍运行中,可按下ctrl+c结束程序'
|
|
997
|
+
else:
|
|
998
|
+
msg = f'主线程执行完毕,但插件[{self.plugin_key}]的服务器线程仍运行中,可按下ctrl+c结束程序'
|
|
999
|
+
|
|
1000
|
+
self.log(msg, 'wait')
|
|
1001
|
+
|
|
1002
|
+
while st.is_alive():
|
|
1003
|
+
try:
|
|
1004
|
+
st.join(timeout=0.5)
|
|
1005
|
+
except KeyboardInterrupt:
|
|
1006
|
+
self.log('收到ctrl+c,结束程序', 'wait')
|
|
1007
|
+
return
|
|
1008
|
+
|
|
1009
|
+
def wait_until_finish(self):
|
|
1010
|
+
self.wait_server_stop(proactive=True)
|
|
1011
|
+
|
|
1012
|
+
@classmethod
|
|
1013
|
+
def build(cls, option: JmOption) -> 'JmOptionPlugin':
|
|
1014
|
+
"""
|
|
1015
|
+
单例模式
|
|
1016
|
+
"""
|
|
1017
|
+
field_name = 'single_instance'
|
|
1018
|
+
|
|
1019
|
+
instance = getattr(cls, field_name, None)
|
|
1020
|
+
if instance is not None:
|
|
1021
|
+
return instance
|
|
1022
|
+
|
|
1023
|
+
with cls.single_instance_lock:
|
|
1024
|
+
instance = getattr(cls, field_name, None)
|
|
1025
|
+
if instance is not None:
|
|
1026
|
+
return instance
|
|
1027
|
+
instance = JmServerPlugin(option)
|
|
1028
|
+
setattr(cls, field_name, instance)
|
|
1029
|
+
return instance
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
class SubscribeAlbumUpdatePlugin(JmOptionPlugin):
|
|
1033
|
+
plugin_key = 'subscribe_album_update'
|
|
1034
|
+
|
|
1035
|
+
def invoke(self,
|
|
1036
|
+
album_photo_dict=None,
|
|
1037
|
+
email_notify=None,
|
|
1038
|
+
download_if_has_update=True,
|
|
1039
|
+
auto_update_after_download=True,
|
|
1040
|
+
) -> None:
|
|
1041
|
+
if album_photo_dict is None:
|
|
1042
|
+
return
|
|
1043
|
+
|
|
1044
|
+
album_photo_dict: Dict
|
|
1045
|
+
for album_id, photo_id in album_photo_dict.copy().items():
|
|
1046
|
+
# check update
|
|
1047
|
+
try:
|
|
1048
|
+
has_update, photo_new_list = self.check_photo_update(album_id, photo_id)
|
|
1049
|
+
except JmcomicException as e:
|
|
1050
|
+
self.log('Exception happened: ' + str(e), 'check_update.error')
|
|
1051
|
+
continue
|
|
1052
|
+
|
|
1053
|
+
if has_update is False:
|
|
1054
|
+
continue
|
|
1055
|
+
|
|
1056
|
+
self.log(f'album={album_id},发现新章节: {photo_new_list},准备开始下载')
|
|
1057
|
+
|
|
1058
|
+
# send email
|
|
1059
|
+
try:
|
|
1060
|
+
if email_notify:
|
|
1061
|
+
SendQQEmailPlugin.build(self.option).invoke(**email_notify)
|
|
1062
|
+
except PluginValidationException:
|
|
1063
|
+
# ignore
|
|
1064
|
+
pass
|
|
1065
|
+
|
|
1066
|
+
# download new photo
|
|
1067
|
+
if has_update and download_if_has_update:
|
|
1068
|
+
self.option.download_photo(photo_new_list)
|
|
1069
|
+
|
|
1070
|
+
if auto_update_after_download:
|
|
1071
|
+
album_photo_dict[album_id] = photo_new_list[-1]
|
|
1072
|
+
self.option.to_file()
|
|
1073
|
+
|
|
1074
|
+
def check_photo_update(self, album_id: str, photo_id: str):
|
|
1075
|
+
client = self.option.new_jm_client()
|
|
1076
|
+
album = client.get_album_detail(album_id)
|
|
1077
|
+
|
|
1078
|
+
photo_new_list = []
|
|
1079
|
+
is_new_photo = False
|
|
1080
|
+
sentinel = int(photo_id)
|
|
1081
|
+
|
|
1082
|
+
for photo in album:
|
|
1083
|
+
if is_new_photo:
|
|
1084
|
+
photo_new_list.append(photo.photo_id)
|
|
1085
|
+
|
|
1086
|
+
if int(photo.photo_id) == sentinel:
|
|
1087
|
+
is_new_photo = True
|
|
1088
|
+
|
|
1089
|
+
return len(photo_new_list) != 0, photo_new_list
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
class SkipPhotoWithFewImagesPlugin(JmOptionPlugin):
|
|
1093
|
+
plugin_key = 'skip_photo_with_few_images'
|
|
1094
|
+
|
|
1095
|
+
def invoke(self,
|
|
1096
|
+
at_least_image_count: int,
|
|
1097
|
+
photo: Optional[JmPhotoDetail] = None,
|
|
1098
|
+
image: Optional[JmImageDetail] = None,
|
|
1099
|
+
album: Optional[JmAlbumDetail] = None,
|
|
1100
|
+
**kwargs
|
|
1101
|
+
):
|
|
1102
|
+
self.try_mark_photo_skip_and_log(photo, at_least_image_count)
|
|
1103
|
+
if image is not None:
|
|
1104
|
+
self.try_mark_photo_skip_and_log(image.from_photo, at_least_image_count)
|
|
1105
|
+
|
|
1106
|
+
def try_mark_photo_skip_and_log(self, photo: JmPhotoDetail, at_least_image_count: int):
|
|
1107
|
+
if photo is None:
|
|
1108
|
+
return
|
|
1109
|
+
|
|
1110
|
+
if len(photo) >= at_least_image_count:
|
|
1111
|
+
return
|
|
1112
|
+
|
|
1113
|
+
self.log(f'跳过下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}]),'
|
|
1114
|
+
f'因为其图片数: {len(photo)} < {at_least_image_count} (at_least_image_count)')
|
|
1115
|
+
photo.skip = True
|
|
1116
|
+
|
|
1117
|
+
@classmethod
|
|
1118
|
+
@field_cache() # 单例
|
|
1119
|
+
def build(cls, option: JmOption) -> 'JmOptionPlugin':
|
|
1120
|
+
return super().build(option)
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
class DeleteDuplicatedFilesPlugin(JmOptionPlugin):
|
|
1124
|
+
"""
|
|
1125
|
+
https://github.com/hect0x7/JMComic-Crawler-Python/issues/244
|
|
1126
|
+
"""
|
|
1127
|
+
plugin_key = 'delete_duplicated_files'
|
|
1128
|
+
|
|
1129
|
+
@classmethod
|
|
1130
|
+
def calculate_md5(cls, file_path):
|
|
1131
|
+
import hashlib
|
|
1132
|
+
|
|
1133
|
+
"""计算文件的MD5哈希值"""
|
|
1134
|
+
hash_md5 = hashlib.md5()
|
|
1135
|
+
with open(file_path, "rb") as f:
|
|
1136
|
+
for chunk in iter(lambda: f.read(4096), b""):
|
|
1137
|
+
hash_md5.update(chunk)
|
|
1138
|
+
return hash_md5.hexdigest()
|
|
1139
|
+
|
|
1140
|
+
@classmethod
|
|
1141
|
+
def find_duplicate_files(cls, root_folder):
|
|
1142
|
+
"""递归读取文件夹下所有文件并计算MD5出现次数"""
|
|
1143
|
+
import os
|
|
1144
|
+
from collections import defaultdict
|
|
1145
|
+
md5_dict = defaultdict(list)
|
|
1146
|
+
|
|
1147
|
+
for root, _, files in os.walk(root_folder):
|
|
1148
|
+
for file in files:
|
|
1149
|
+
file_path = os.path.join(root, file)
|
|
1150
|
+
file_md5 = cls.calculate_md5(file_path)
|
|
1151
|
+
md5_dict[file_md5].append(file_path)
|
|
1152
|
+
|
|
1153
|
+
return md5_dict
|
|
1154
|
+
|
|
1155
|
+
def invoke(self,
|
|
1156
|
+
limit,
|
|
1157
|
+
album=None,
|
|
1158
|
+
downloader=None,
|
|
1159
|
+
delete_original_file=True,
|
|
1160
|
+
**kwargs,
|
|
1161
|
+
) -> None:
|
|
1162
|
+
if album is None:
|
|
1163
|
+
return
|
|
1164
|
+
|
|
1165
|
+
self.delete_original_file = delete_original_file
|
|
1166
|
+
# 获取到下载本子所在根目录
|
|
1167
|
+
root_folder = self.option.dir_rule.decide_album_root_dir(album)
|
|
1168
|
+
self.find_duplicated_files_and_delete(limit, root_folder, album)
|
|
1169
|
+
|
|
1170
|
+
def find_duplicated_files_and_delete(self, limit: int, root_folder: str, album: Optional[JmAlbumDetail] = None):
|
|
1171
|
+
md5_dict = self.find_duplicate_files(root_folder)
|
|
1172
|
+
# 打印MD5出现次数大于等于limit的文件
|
|
1173
|
+
for md5, paths in md5_dict.items():
|
|
1174
|
+
if len(paths) >= limit:
|
|
1175
|
+
prefix = '' if album is None else f'({album.album_id}) '
|
|
1176
|
+
message = [prefix + f'MD5: {md5} 出现次数: {len(paths)}'] + \
|
|
1177
|
+
[f' {path}' for path in paths]
|
|
1178
|
+
self.log('\n'.join(message))
|
|
1179
|
+
self.execute_deletion(paths)
|
|
1180
|
+
|
|
1181
|
+
|
|
1182
|
+
class ReplacePathStringPlugin(JmOptionPlugin):
|
|
1183
|
+
plugin_key = 'replace_path_string'
|
|
1184
|
+
|
|
1185
|
+
def invoke(self,
|
|
1186
|
+
replace: Dict[str, str],
|
|
1187
|
+
):
|
|
1188
|
+
if not replace:
|
|
1189
|
+
return
|
|
1190
|
+
|
|
1191
|
+
old_decide_dir = self.option.decide_image_save_dir
|
|
1192
|
+
|
|
1193
|
+
def new_decide_dir(photo, ensure_exists=True) -> str:
|
|
1194
|
+
original_path: str = old_decide_dir(photo, False)
|
|
1195
|
+
for k, v in replace.items():
|
|
1196
|
+
original_path = original_path.replace(k, v)
|
|
1197
|
+
|
|
1198
|
+
if ensure_exists:
|
|
1199
|
+
JmcomicText.try_mkdir(original_path)
|
|
1200
|
+
|
|
1201
|
+
return original_path
|
|
1202
|
+
|
|
1203
|
+
self.option.decide_image_save_dir = new_decide_dir
|