jmcomic 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
jmcomic/jm_plugin.py ADDED
@@ -0,0 +1,1203 @@
1
+ """
2
+ 该文件存放的是option插件
3
+ """
4
+
5
+ from .jm_option import *
6
+
7
+
8
+ class PluginValidationException(Exception):
9
+
10
+ def __init__(self, plugin: 'JmOptionPlugin', msg: str):
11
+ self.plugin = plugin
12
+ self.msg = msg
13
+
14
+
15
+ class JmOptionPlugin:
16
+ plugin_key: str
17
+
18
+ def __init__(self, option: JmOption):
19
+ self.option = option
20
+ self.log_enable = True
21
+ self.delete_original_file = False
22
+
23
+ def invoke(self, **kwargs) -> None:
24
+ """
25
+ 执行插件的功能
26
+ :param kwargs: 给插件的参数
27
+ """
28
+ raise NotImplementedError
29
+
30
+ @classmethod
31
+ def build(cls, option: JmOption) -> 'JmOptionPlugin':
32
+ """
33
+ 创建插件实例
34
+ :param option: JmOption对象
35
+ """
36
+ return cls(option)
37
+
38
+ def log(self, msg, topic=None):
39
+ if self.log_enable is not True:
40
+ return
41
+
42
+ jm_log(
43
+ topic=f'plugin.{self.plugin_key}' + (f'.{topic}' if topic is not None else ''),
44
+ msg=msg
45
+ )
46
+
47
+ def require_param(self, case: Any, msg: str):
48
+ """
49
+ 专门用于校验参数的方法,会抛出特定异常,由option拦截根据策略进行处理
50
+
51
+ :param case: 条件
52
+ :param msg: 报错信息
53
+ """
54
+ if case:
55
+ return
56
+
57
+ raise PluginValidationException(self, msg)
58
+
59
+ def warning_lib_not_install(self, lib: str, throw=False):
60
+ msg = (f'插件`{self.plugin_key}`依赖库: {lib},请先安装{lib}再使用。'
61
+ f'安装命令: [pip install {lib}]')
62
+ import warnings
63
+ warnings.warn(msg)
64
+ self.require_param(throw, msg)
65
+
66
+ def execute_deletion(self, paths: List[str]):
67
+ """
68
+ 删除文件和文件夹
69
+ :param paths: 路径列表
70
+ """
71
+ if self.delete_original_file is not True:
72
+ return
73
+
74
+ for p in paths:
75
+ if file_not_exists(p):
76
+ continue
77
+
78
+ if os.path.isdir(p):
79
+ if os.listdir(p):
80
+ self.log(f'文件夹中存在非本次下载的文件,请手动删除文件夹内的文件: {p}', 'remove.ignore')
81
+ continue
82
+ os.rmdir(p)
83
+ self.log(f'删除文件夹: {p}', 'remove')
84
+ else:
85
+ os.remove(p)
86
+ self.log(f'删除原文件: {p}', 'remove')
87
+
88
+ # noinspection PyMethodMayBeStatic
89
+ def execute_cmd(self, cmd):
90
+ """
91
+ 执行shell命令,这里采用简单的实现
92
+ :param cmd: shell命令
93
+ """
94
+ return os.system(cmd)
95
+
96
+ # noinspection PyMethodMayBeStatic
97
+ def execute_multi_line_cmd(self, cmd: str):
98
+ import subprocess
99
+ subprocess.run(cmd, shell=True, check=True)
100
+
101
+ def enter_wait_list(self):
102
+ self.option.need_wait_plugins.append(self)
103
+
104
+ def leave_wait_list(self):
105
+ self.option.need_wait_plugins.remove(self)
106
+
107
+ def wait_until_finish(self):
108
+ pass
109
+
110
+ # noinspection PyMethodMayBeStatic
111
+ def decide_filepath(self,
112
+ album: Optional[JmAlbumDetail],
113
+ photo: Optional[JmPhotoDetail],
114
+ filename_rule: str, suffix: str, base_dir: Optional[str],
115
+ dir_rule_dict: Optional[dict]
116
+ ):
117
+ """
118
+ 根据规则计算一个文件的全路径
119
+
120
+ 参数 dir_rule_dict 优先级最高,
121
+ 如果 dir_rule_dict 不为空,优先用 dir_rule_dict
122
+ 否则使用 base_dir + filename_rule + suffix
123
+ """
124
+ filepath: str
125
+ base_dir: str
126
+ if dir_rule_dict is not None:
127
+ dir_rule = DirRule(**dir_rule_dict)
128
+ filepath = dir_rule.apply_rule_to_path(album, photo)
129
+ base_dir = os.path.dirname(filepath)
130
+ else:
131
+ base_dir = base_dir or os.getcwd()
132
+ filepath = os.path.join(base_dir, DirRule.apply_rule_to_filename(album, photo, filename_rule) + fix_suffix(suffix))
133
+
134
+ mkdir_if_not_exists(base_dir)
135
+ return filepath
136
+
137
+
138
+ class JmLoginPlugin(JmOptionPlugin):
139
+ """
140
+ 功能:登录禁漫,并保存登录后的cookies,让所有client都带上此cookies
141
+ """
142
+ plugin_key = 'login'
143
+
144
+ def invoke(self,
145
+ username: str,
146
+ password: str,
147
+ impl=None,
148
+ ) -> None:
149
+ self.require_param(username, '用户名不能为空')
150
+ self.require_param(password, '密码不能为空')
151
+
152
+ client = self.option.build_jm_client(impl=impl)
153
+ client.login(username, password)
154
+
155
+ cookies = dict(client['cookies'])
156
+ self.option.update_cookies(cookies)
157
+ JmModuleConfig.APP_COOKIES = cookies
158
+
159
+ self.log('登录成功')
160
+
161
+
162
+ class UsageLogPlugin(JmOptionPlugin):
163
+ plugin_key = 'usage_log'
164
+
165
+ def invoke(self, **kwargs) -> None:
166
+ import threading
167
+ t = threading.Thread(
168
+ target=self.monitor_resource_usage,
169
+ kwargs=kwargs,
170
+ daemon=True,
171
+ )
172
+ t.start()
173
+
174
+ self.set_thread_as_option_attr(t)
175
+
176
+ def set_thread_as_option_attr(self, t):
177
+ """
178
+ 线程留痕
179
+ """
180
+ name = f'thread_{self.plugin_key}'
181
+
182
+ thread_ls: Optional[list] = getattr(self.option, name, None)
183
+ if thread_ls is None:
184
+ setattr(self.option, name, [t])
185
+ else:
186
+ thread_ls.append(t)
187
+
188
+ def monitor_resource_usage(
189
+ self,
190
+ interval=1,
191
+ enable_warning=True,
192
+ warning_cpu_percent=70,
193
+ warning_mem_percent=70,
194
+ warning_thread_count=100,
195
+ ):
196
+ try:
197
+ import psutil
198
+ except ImportError:
199
+ self.warning_lib_not_install('psutil')
200
+ return
201
+
202
+ from time import sleep
203
+ from threading import active_count
204
+ # 获取当前进程
205
+ process = psutil.Process()
206
+
207
+ cpu_percent = None
208
+ # noinspection PyUnusedLocal
209
+ thread_count = None
210
+ # noinspection PyUnusedLocal
211
+ mem_usage = None
212
+
213
+ def warning():
214
+ warning_msg_list = []
215
+ if cpu_percent >= warning_cpu_percent:
216
+ warning_msg_list.append(f'进程占用cpu过高 ({cpu_percent}% >= {warning_cpu_percent}%)')
217
+
218
+ mem_percent = psutil.virtual_memory().percent
219
+ if mem_percent >= warning_mem_percent:
220
+ warning_msg_list.append(f'系统内存占用过高 ({mem_percent}% >= {warning_mem_percent}%)')
221
+
222
+ if thread_count >= warning_thread_count:
223
+ warning_msg_list.append(f'线程数过多 ({thread_count} >= {warning_thread_count})')
224
+
225
+ if len(warning_msg_list) != 0:
226
+ warning_msg_list.insert(0, '硬件占用告警,占用过高可能导致系统卡死!')
227
+ warning_msg_list.append('')
228
+ self.log('\n'.join(warning_msg_list), topic='warning')
229
+
230
+ while True:
231
+ # 获取CPU占用率(0~100)
232
+ cpu_percent = process.cpu_percent()
233
+ # 获取内存占用(MB)
234
+ mem_usage = round(process.memory_info().rss / 1024 / 1024, 2)
235
+ thread_count = active_count()
236
+ # 获取网络占用情况
237
+ # network_info = psutil.net_io_counters()
238
+ # network_bytes_sent = network_info.bytes_sent
239
+ # network_bytes_received = network_info.bytes_recv
240
+
241
+ # 打印信息
242
+ msg = ', '.join([
243
+ f'线程数: {thread_count}',
244
+ f'CPU占用: {cpu_percent}%',
245
+ f'内存占用: {mem_usage}MB',
246
+ # f"发送的字节数: {network_bytes_sent}",
247
+ # f"接收的字节数: {network_bytes_received}",
248
+ ])
249
+ self.log(msg, topic='log')
250
+
251
+ if enable_warning is True:
252
+ # 警告
253
+ warning()
254
+
255
+ # 等待一段时间
256
+ sleep(interval)
257
+
258
+
259
+ class FindUpdatePlugin(JmOptionPlugin):
260
+ """
261
+ 参考: https://github.com/hect0x7/JMComic-Crawler-Python/issues/95
262
+ """
263
+ plugin_key = 'find_update'
264
+
265
+ def invoke(self, **kwargs) -> None:
266
+ self.download_album_with_find_update(kwargs or {})
267
+
268
+ def download_album_with_find_update(self, dic: Dict[str, int]):
269
+ from .api import download_album
270
+ from .jm_downloader import JmDownloader
271
+
272
+ # 带入漫画id, 章节id(第x章),寻找该漫画下第x章节後的所有章节Id
273
+ def find_update(album: JmAlbumDetail):
274
+ if album.album_id not in dic:
275
+ return album
276
+
277
+ photo_ls = []
278
+ photo_begin = int(dic[album.album_id])
279
+ is_new_photo = False
280
+
281
+ for photo in album:
282
+ if is_new_photo:
283
+ photo_ls.append(photo)
284
+
285
+ if int(photo.photo_id) == photo_begin:
286
+ is_new_photo = True
287
+
288
+ return photo_ls
289
+
290
+ class FindUpdateDownloader(JmDownloader):
291
+ def do_filter(self, detail):
292
+ if not detail.is_album():
293
+ return detail
294
+
295
+ detail: JmAlbumDetail
296
+ return find_update(detail)
297
+
298
+ # 调用下载api,指定option和downloader
299
+ download_album(
300
+ jm_album_id=dic.keys(),
301
+ option=self.option,
302
+ downloader=FindUpdateDownloader,
303
+ )
304
+
305
+
306
+ class ZipPlugin(JmOptionPlugin):
307
+ """
308
+ 感谢zip加密功能的贡献者:
309
+ - AXIS5 a.k.a AXIS5Hacker (https://github.com/hect0x7/JMComic-Crawler-Python/pull/375)
310
+ """
311
+
312
+ plugin_key = 'zip'
313
+
314
+ # noinspection PyAttributeOutsideInit
315
+ def invoke(self,
316
+ downloader,
317
+ album: JmAlbumDetail = None,
318
+ photo: JmPhotoDetail = None,
319
+ delete_original_file=False,
320
+ level='photo',
321
+ filename_rule='Ptitle',
322
+ suffix='zip',
323
+ zip_dir='./',
324
+ dir_rule=None,
325
+ encrypt=None,
326
+ ) -> None:
327
+
328
+ from .jm_downloader import JmDownloader
329
+ downloader: JmDownloader
330
+ self.downloader = downloader
331
+ self.level = level
332
+ self.delete_original_file = delete_original_file
333
+
334
+ # 确保压缩文件所在文件夹存在
335
+ zip_dir = JmcomicText.parse_to_abspath(zip_dir)
336
+ mkdir_if_not_exists(zip_dir)
337
+
338
+ path_to_delete = []
339
+ photo_dict = self.get_downloaded_photo(downloader, album, photo)
340
+
341
+ if level == 'album':
342
+ zip_path = self.decide_filepath(album, None, filename_rule, suffix, zip_dir, dir_rule)
343
+ self.zip_album(album, photo_dict, zip_path, path_to_delete, encrypt)
344
+
345
+ elif level == 'photo':
346
+ for photo, image_list in photo_dict.items():
347
+ zip_path = self.decide_filepath(photo.from_album, photo, filename_rule, suffix, zip_dir, dir_rule)
348
+ self.zip_photo(photo, image_list, zip_path, path_to_delete, encrypt)
349
+
350
+ else:
351
+ ExceptionTool.raises(f'Not Implemented Zip Level: {level}')
352
+
353
+ self.after_zip(path_to_delete)
354
+
355
+ # noinspection PyMethodMayBeStatic
356
+ def get_downloaded_photo(self, downloader, album, photo):
357
+ return (
358
+ downloader.download_success_dict[album]
359
+ if album is not None # after_album
360
+ else downloader.download_success_dict[photo.from_album] # after_photo
361
+ )
362
+
363
+ def zip_photo(self, photo, image_list: list, zip_path: str, path_to_delete, encrypt_dict):
364
+ """
365
+ 压缩photo文件夹
366
+ """
367
+ photo_dir = self.option.decide_image_save_dir(photo) \
368
+ if len(image_list) == 0 \
369
+ else os.path.dirname(image_list[0][0])
370
+
371
+ with self.open_zip_file(zip_path, encrypt_dict) as f:
372
+ for file in files_of_dir(photo_dir):
373
+ abspath = os.path.join(photo_dir, file)
374
+ relpath = os.path.relpath(abspath, photo_dir)
375
+ f.write(abspath, relpath)
376
+
377
+ self.log(f'压缩章节[{photo.photo_id}]成功 → {zip_path}', 'finish')
378
+ path_to_delete.append(self.unified_path(photo_dir))
379
+
380
+ @staticmethod
381
+ def unified_path(f):
382
+ return fix_filepath(f, os.path.isdir(f))
383
+
384
+ def zip_album(self, album, photo_dict: dict, zip_path, path_to_delete, encrypt_dict):
385
+ """
386
+ 压缩album文件夹
387
+ """
388
+
389
+ album_dir = self.option.dir_rule.decide_album_root_dir(album)
390
+ with self.open_zip_file(zip_path, encrypt_dict) as f:
391
+ for photo in photo_dict.keys():
392
+ # 定位到章节所在文件夹
393
+ photo_dir = self.unified_path(self.option.decide_image_save_dir(photo))
394
+ # 章节文件夹标记为删除
395
+ path_to_delete.append(photo_dir)
396
+ for file in files_of_dir(photo_dir):
397
+ abspath = os.path.join(photo_dir, file)
398
+ relpath = os.path.relpath(abspath, album_dir)
399
+ f.write(abspath, relpath)
400
+ self.log(f'压缩本子[{album.album_id}]成功 → {zip_path}', 'finish')
401
+
402
+ def after_zip(self, path_to_delete: List[str]):
403
+ # 删除所有原文件
404
+ dirs = sorted(path_to_delete, reverse=True)
405
+ image_paths = [
406
+ path
407
+ for photo_dict in self.downloader.download_success_dict.values()
408
+ for image_list in photo_dict.values()
409
+ for path, image in image_list
410
+ ]
411
+ self.execute_deletion(image_paths)
412
+ self.execute_deletion(dirs)
413
+
414
+ # noinspection PyMethodMayBeStatic
415
+ @classmethod
416
+ def generate_random_str(cls, random_length) -> str:
417
+ """
418
+ 自动生成随机字符密码,长度由randomlength指定
419
+ """
420
+ import random
421
+
422
+ random_str = ''
423
+ base_str = r'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='
424
+ base_length = len(base_str) - 1
425
+ for _ in range(random_length):
426
+ random_str += base_str[random.randint(0, base_length)]
427
+ return random_str
428
+
429
+ def open_zip_file(self, zip_path: str, encrypt_dict: Optional[dict]):
430
+ if encrypt_dict is None:
431
+ import zipfile
432
+ return zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
433
+
434
+ password, is_random = self.decide_password(encrypt_dict, zip_path)
435
+ if encrypt_dict.get('impl', '') == '7z':
436
+ try:
437
+ # noinspection PyUnresolvedReferences
438
+ import py7zr
439
+ except ImportError:
440
+ self.warning_lib_not_install('py7zr', True)
441
+
442
+ # noinspection PyUnboundLocalVariable
443
+ filters = [{'id': py7zr.FILTER_COPY}]
444
+ return py7zr.SevenZipFile(zip_path, mode='w', password=password, filters=filters, header_encryption=True)
445
+ else:
446
+ try:
447
+ # noinspection PyUnresolvedReferences
448
+ import pyzipper
449
+ except ImportError:
450
+ self.warning_lib_not_install('pyzipper', True)
451
+
452
+ # noinspection PyUnboundLocalVariable
453
+ aes_zip_file = pyzipper.AESZipFile(zip_path, "w", pyzipper.ZIP_DEFLATED)
454
+ aes_zip_file.setencryption(pyzipper.WZ_AES, nbits=128)
455
+ password_bytes = str.encode(password)
456
+ aes_zip_file.setpassword(password_bytes)
457
+ if is_random:
458
+ aes_zip_file.comment = password_bytes
459
+ return aes_zip_file
460
+
461
+ def decide_password(self, encrypt_dict: dict, zip_path: str):
462
+ encrypt_type = encrypt_dict.get('type', '')
463
+ is_random = False
464
+
465
+ if encrypt_type == 'random':
466
+ is_random = True
467
+ password = self.generate_random_str(48)
468
+ self.log(f'生成随机密码: [{password}] → [{zip_path}]', 'encrypt')
469
+ else:
470
+ password = str(encrypt_dict['password'])
471
+ self.log(f'使用指定密码: [{password}] → [{zip_path}]', 'encrypt')
472
+
473
+ return password, is_random
474
+
475
+
476
+ class ClientProxyPlugin(JmOptionPlugin):
477
+ plugin_key = 'client_proxy'
478
+
479
+ def invoke(self,
480
+ proxy_client_key,
481
+ whitelist=None,
482
+ **clazz_init_kwargs,
483
+ ) -> None:
484
+ if whitelist is not None:
485
+ whitelist = set(whitelist)
486
+
487
+ proxy_clazz = JmModuleConfig.client_impl_class(proxy_client_key)
488
+ new_jm_client: Callable = self.option.new_jm_client
489
+
490
+ def hook_new_jm_client(*args, **kwargs):
491
+ client = new_jm_client(*args, **kwargs)
492
+ if whitelist is not None and client.client_key not in whitelist:
493
+ return client
494
+
495
+ self.log(f'proxy client {client} with {proxy_clazz}')
496
+ return proxy_clazz(client, **clazz_init_kwargs)
497
+
498
+ self.option.new_jm_client = hook_new_jm_client
499
+
500
+
501
+ class ImageSuffixFilterPlugin(JmOptionPlugin):
502
+ plugin_key = 'image_suffix_filter'
503
+
504
+ def invoke(self,
505
+ allowed_orig_suffix=None,
506
+ ) -> None:
507
+ if allowed_orig_suffix is None:
508
+ return
509
+
510
+ allowed_suffix_set = set(fix_suffix(suffix) for suffix in allowed_orig_suffix)
511
+
512
+ option_decide_cache = self.option.decide_download_cache
513
+
514
+ def apply_filter_then_decide_cache(image: JmImageDetail):
515
+ if image.img_file_suffix not in allowed_suffix_set:
516
+ self.log(f'跳过下载图片: {image.tag},'
517
+ f'因为其后缀\'{image.img_file_suffix}\'不在允许的后缀集合{allowed_suffix_set}内')
518
+ image.skip = True
519
+
520
+ # let option decide
521
+ return option_decide_cache(image)
522
+
523
+ self.option.decide_download_cache = apply_filter_then_decide_cache
524
+
525
+
526
+ class SendQQEmailPlugin(JmOptionPlugin):
527
+ plugin_key = 'send_qq_email'
528
+
529
+ def invoke(self,
530
+ msg_from,
531
+ msg_to,
532
+ password,
533
+ title,
534
+ content,
535
+ album=None,
536
+ downloader=None,
537
+ ) -> None:
538
+ self.require_param(msg_from and msg_to and password, '发件人、收件人、授权码都不能为空')
539
+
540
+ from common import EmailConfig
541
+ econfig = EmailConfig(msg_from, msg_to, password)
542
+ epostman = econfig.create_email_postman()
543
+ epostman.send(content, title)
544
+
545
+ self.log('Email sent successfully')
546
+
547
+
548
+ class LogTopicFilterPlugin(JmOptionPlugin):
549
+ plugin_key = 'log_topic_filter'
550
+
551
+ def invoke(self, whitelist) -> None:
552
+ if whitelist is not None:
553
+ whitelist = set(whitelist)
554
+
555
+ old_jm_log = JmModuleConfig.EXECUTOR_LOG
556
+
557
+ def new_jm_log(topic, msg):
558
+ if whitelist is not None and topic not in whitelist:
559
+ return
560
+
561
+ old_jm_log(topic, msg)
562
+
563
+ JmModuleConfig.EXECUTOR_LOG = new_jm_log
564
+
565
+
566
+ class AutoSetBrowserCookiesPlugin(JmOptionPlugin):
567
+ plugin_key = 'auto_set_browser_cookies'
568
+
569
+ accepted_cookies_keys = str_to_set('''
570
+ yuo1
571
+ remember_id
572
+ remember
573
+ ''')
574
+
575
+ def invoke(self,
576
+ browser: str,
577
+ domain: str,
578
+ ) -> None:
579
+ """
580
+ 坑点预警:由于禁漫需要校验同一设备,使用该插件需要配置自己浏览器的headers,例如
581
+
582
+ ```yml
583
+ client:
584
+ postman:
585
+ meta_data:
586
+ headers: {
587
+ # 浏览器headers
588
+ }
589
+
590
+ # 插件配置如下:
591
+ plugins:
592
+ after_init:
593
+ - plugin: auto_set_browser_cookies
594
+ kwargs:
595
+ browser: chrome
596
+ domain: 18comic.vip
597
+ ```
598
+
599
+ :param browser: chrome/edge/...
600
+ :param domain: 18comic.vip/...
601
+ :return: cookies
602
+ """
603
+ cookies, e = get_browser_cookies(browser, domain, safe=True)
604
+
605
+ if cookies is None:
606
+ if isinstance(e, ImportError):
607
+ self.warning_lib_not_install('browser_cookie3')
608
+ else:
609
+ self.log('获取浏览器cookies失败,请关闭浏览器重试')
610
+ return
611
+
612
+ self.option.update_cookies(
613
+ {k: v for k, v in cookies.items() if k in self.accepted_cookies_keys}
614
+ )
615
+ self.log('获取浏览器cookies成功')
616
+
617
+
618
+ # noinspection PyMethodMayBeStatic
619
+ class FavoriteFolderExportPlugin(JmOptionPlugin):
620
+ plugin_key = 'favorite_folder_export'
621
+
622
+ # noinspection PyAttributeOutsideInit
623
+ def invoke(self,
624
+ save_dir=None,
625
+ zip_enable=False,
626
+ zip_filepath=None,
627
+ zip_password=None,
628
+ delete_original_file=False,
629
+ ):
630
+ self.save_dir = os.path.abspath(save_dir if save_dir is not None else (os.getcwd() + '/export/'))
631
+ self.zip_enable = zip_enable
632
+ self.zip_filepath = os.path.abspath(zip_filepath)
633
+ self.zip_password = zip_password
634
+ self.delete_original_file = delete_original_file
635
+ self.files = []
636
+
637
+ mkdir_if_not_exists(self.save_dir)
638
+ mkdir_if_not_exists(of_dir_path(self.zip_filepath))
639
+
640
+ self.main()
641
+
642
+ def main(self):
643
+ cl = self.option.build_jm_client()
644
+ # noinspection PyAttributeOutsideInit
645
+ self.cl = cl
646
+ page = cl.favorite_folder()
647
+
648
+ # 获取所有的收藏夹
649
+ folders = {fid: fname for fid, fname in page.iter_folder_id_name()}
650
+ # 加上特殊收藏栏【全部】
651
+ folders.setdefault('0', '全部')
652
+
653
+ # 一个收藏夹一个线程,导出收藏夹数据到文件
654
+ multi_thread_launcher(
655
+ iter_objs=folders.items(),
656
+ apply_each_obj_func=self.handle_folder,
657
+ )
658
+
659
+ if not self.zip_enable:
660
+ return
661
+
662
+ # 压缩导出的文件
663
+ self.require_param(self.zip_filepath, '如果开启zip,请指定zip_filepath参数(压缩文件保存路径)')
664
+
665
+ if self.zip_password is None:
666
+ self.zip_folder_without_password(self.files, self.zip_filepath)
667
+ else:
668
+ self.zip_with_password()
669
+
670
+ self.execute_deletion(self.files)
671
+
672
+ def handle_folder(self, fid: str, fname: str):
673
+ self.log(f'【收藏夹: {fname}, fid: {fid}】开始获取数据')
674
+
675
+ # 获取收藏夹数据
676
+ page_data = self.fetch_folder_page_data(fid)
677
+
678
+ # 序列化到文件
679
+ filepath = self.save_folder_page_data_to_file(page_data, fid, fname)
680
+
681
+ if filepath is None:
682
+ self.log(f'【收藏夹: {fname}, fid: {fid}】收藏夹无数据')
683
+ return
684
+
685
+ self.log(f'【收藏夹: {fname}, fid: {fid}】保存文件成功 → [{filepath}]')
686
+ self.files.append(filepath)
687
+
688
+ def fetch_folder_page_data(self, fid):
689
+ # 一页一页获取,不使用并行
690
+ page_data = list(self.cl.favorite_folder_gen(folder_id=fid))
691
+ return page_data
692
+
693
+ def save_folder_page_data_to_file(self, page_data: List[JmFavoritePage], fid: str, fname: str):
694
+ from os import path
695
+ filepath = path.abspath(path.join(self.save_dir, fix_windir_name(f'【{fid}】{fname}.csv')))
696
+
697
+ data = []
698
+ for page in page_data:
699
+ for aid, extra in page.content:
700
+ data.append(
701
+ (aid, extra.get('author', '') or JmModuleConfig.DEFAULT_AUTHOR, extra['name'])
702
+ )
703
+
704
+ if len(data) == 0:
705
+ return
706
+
707
+ with open(filepath, 'w', encoding='utf-8') as f:
708
+ f.write('id,author,name\n')
709
+ for item in data:
710
+ f.write(','.join(item) + '\n')
711
+
712
+ return filepath
713
+
714
+ def zip_folder_without_password(self, files, zip_path):
715
+ """
716
+ 压缩文件夹中的文件并设置密码
717
+
718
+ :param files: 要压缩的文件的绝对路径的列表
719
+ :param zip_path: 压缩文件的保存路径
720
+ """
721
+ import zipfile
722
+
723
+ with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
724
+ # 获取文件夹中的文件列表并将其添加到 ZIP 文件中
725
+ for file in files:
726
+ zipf.write(file, arcname=of_file_name(file))
727
+
728
+ def zip_with_password(self):
729
+ # 构造shell命令
730
+ cmd_list = f'''
731
+ cd {self.save_dir}
732
+ 7z a "{self.zip_filepath}" "./" -p{self.zip_password} -mhe=on > "../7z_output.txt"
733
+
734
+ '''
735
+ self.log(f'运行命令: {cmd_list}')
736
+
737
+ # 执行
738
+ self.execute_multi_line_cmd(cmd_list)
739
+
740
+
741
+ class Img2pdfPlugin(JmOptionPlugin):
742
+ plugin_key = 'img2pdf'
743
+
744
+ def invoke(self,
745
+ photo: JmPhotoDetail = None,
746
+ album: JmAlbumDetail = None,
747
+ downloader=None,
748
+ pdf_dir=None,
749
+ filename_rule='Pid',
750
+ dir_rule=None,
751
+ delete_original_file=False,
752
+ **kwargs,
753
+ ):
754
+ if photo is None and album is None:
755
+ jm_log('wrong_usage', 'img2pdf必须运行在after_photo或after_album时')
756
+
757
+ try:
758
+ import img2pdf
759
+ except ImportError:
760
+ self.warning_lib_not_install('img2pdf')
761
+ return
762
+
763
+ self.delete_original_file = delete_original_file
764
+
765
+ # 处理生成的pdf文件的路径
766
+ pdf_filepath = self.decide_filepath(album, photo, filename_rule, 'pdf', pdf_dir, dir_rule)
767
+
768
+ # 调用 img2pdf 把 photo_dir 下的所有图片转为pdf
769
+ img_path_ls, img_dir_ls = self.write_img_2_pdf(pdf_filepath, album, photo)
770
+ self.log(f'Convert Successfully: JM{album or photo} → {pdf_filepath}')
771
+
772
+ # 执行删除
773
+ img_path_ls += img_dir_ls
774
+ self.execute_deletion(img_path_ls)
775
+
776
+ def write_img_2_pdf(self, pdf_filepath, album: JmAlbumDetail, photo: JmPhotoDetail):
777
+ import img2pdf
778
+
779
+ if album is None:
780
+ img_dir_ls = [self.option.decide_image_save_dir(photo)]
781
+ else:
782
+ img_dir_ls = [self.option.decide_image_save_dir(photo) for photo in album]
783
+
784
+ img_path_ls = []
785
+
786
+ for img_dir in img_dir_ls:
787
+ imgs = files_of_dir(img_dir)
788
+ if not imgs:
789
+ continue
790
+ img_path_ls += imgs
791
+
792
+ if len(img_path_ls) == 0:
793
+ self.log(f'所有文件夹都不存在图片,无法生成pdf:{img_dir_ls}', 'error')
794
+
795
+ with open(pdf_filepath, 'wb') as f:
796
+ f.write(img2pdf.convert(img_path_ls))
797
+
798
+ return img_path_ls, img_dir_ls
799
+
800
+
801
+ class LongImgPlugin(JmOptionPlugin):
802
+ plugin_key = 'long_img'
803
+
804
+ def invoke(self,
805
+ photo: JmPhotoDetail = None,
806
+ album: JmAlbumDetail = None,
807
+ downloader=None,
808
+ img_dir=None,
809
+ filename_rule='Pid',
810
+ delete_original_file=False,
811
+ dir_rule=None,
812
+ **kwargs,
813
+ ):
814
+ if photo is None and album is None:
815
+ jm_log('wrong_usage', 'long_img必须运行在after_photo或after_album时')
816
+
817
+ try:
818
+ from PIL import Image
819
+ except ImportError:
820
+ self.warning_lib_not_install('PIL')
821
+ return
822
+
823
+ self.delete_original_file = delete_original_file
824
+
825
+ # 处理生成的长图文件的路径
826
+ long_img_path = self.decide_filepath(album, photo, filename_rule, 'png', img_dir, dir_rule)
827
+
828
+ # 调用 PIL 把 photo_dir 下的所有图片合并为长图
829
+ img_path_ls = self.write_img_2_long_img(long_img_path, album, photo)
830
+ self.log(f'Convert Successfully: JM{album or photo} → {long_img_path}')
831
+
832
+ # 执行删除
833
+ self.execute_deletion(img_path_ls)
834
+
835
+ def write_img_2_long_img(self, long_img_path, album: JmAlbumDetail, photo: JmPhotoDetail) -> List[str]:
836
+ import itertools
837
+ from PIL import Image
838
+
839
+ if album is None:
840
+ img_dir_items = [self.option.decide_image_save_dir(photo)]
841
+ else:
842
+ img_dir_items = [self.option.decide_image_save_dir(photo) for photo in album]
843
+
844
+ img_paths = itertools.chain(*map(files_of_dir, img_dir_items))
845
+ img_paths = list(filter(lambda x: not x.startswith('.'), img_paths)) # 过滤系统文件
846
+
847
+ images = self.open_images(img_paths)
848
+
849
+ try:
850
+ resample_method = Image.Resampling.LANCZOS
851
+ except AttributeError:
852
+ resample_method = Image.LANCZOS
853
+
854
+ min_img_width = min(img.width for img in images)
855
+ total_height = 0
856
+ for i, img in enumerate(images):
857
+ if img.width > min_img_width:
858
+ images[i] = img.resize((min_img_width, int(img.height * min_img_width / img.width)),
859
+ resample=resample_method)
860
+ total_height += images[i].height
861
+
862
+ long_img = Image.new('RGB', (min_img_width, total_height))
863
+ y_offset = 0
864
+ for img in images:
865
+ long_img.paste(img, (0, y_offset))
866
+ y_offset += img.height
867
+
868
+ long_img.save(long_img_path)
869
+ for img in images:
870
+ img.close()
871
+
872
+ return img_paths
873
+
874
+ def open_images(self, img_paths: List[str]):
875
+ images = []
876
+ for img_path in img_paths:
877
+ try:
878
+ img = Image.open(img_path)
879
+ images.append(img)
880
+ except IOError as e:
881
+ self.log(f"Failed to open image {img_path}: {e}", 'error')
882
+ return images
883
+
884
+
885
+ class JmServerPlugin(JmOptionPlugin):
886
+ plugin_key = 'jm_server'
887
+
888
+ default_run_kwargs = {
889
+ 'host': '0.0.0.0',
890
+ 'port': '80',
891
+ 'debug': False,
892
+ }
893
+
894
+ from threading import Lock
895
+ single_instance_lock = Lock()
896
+
897
+ def __init__(self, option: JmOption):
898
+ super().__init__(option)
899
+ self.run_server_lock = Lock()
900
+ self.running = False
901
+ self.server_thread: Optional[Thread] = None
902
+
903
+ def invoke(self,
904
+ password='',
905
+ base_dir=None,
906
+ album=None,
907
+ photo=None,
908
+ downloader=None,
909
+ run=None,
910
+ **kwargs
911
+ ):
912
+ """
913
+
914
+ :param password: 密码
915
+ :param base_dir: 初始访问服务器的根路径
916
+ :param album: 为了支持 after_album 这种调用时机
917
+ :param photo: 为了支持 after_album 这种调用时机
918
+ :param downloader: 为了支持 after_album 这种调用时机
919
+ :param run: 用于启动服务器: app.run(**run_kwargs)
920
+ :param kwargs: 用于JmServer构造函数: JmServer(base_dir, password, **kwargs)
921
+ """
922
+
923
+ if base_dir is None:
924
+ base_dir = self.option.dir_rule.base_dir
925
+
926
+ if run is None:
927
+ run = self.default_run_kwargs
928
+ else:
929
+ base_run_kwargs = self.default_run_kwargs.copy()
930
+ base_run_kwargs.update(run)
931
+ run = base_run_kwargs
932
+
933
+ if self.running is True:
934
+ return
935
+
936
+ with self.run_server_lock:
937
+ if self.running is True:
938
+ return
939
+
940
+ # 服务器的代码位于一个独立库:plugin_jm_server,需要独立安装
941
+ # 源代码仓库:https://github.com/hect0x7/plugin-jm-server
942
+ try:
943
+ # noinspection PyUnresolvedReferences
944
+ import plugin_jm_server
945
+ self.log(f'当前使用plugin_jm_server版本: {plugin_jm_server.__version__}')
946
+ except ImportError:
947
+ self.warning_lib_not_install('plugin_jm_server')
948
+ return
949
+
950
+ # 核心函数,启动服务器,会阻塞当前线程
951
+ def blocking_run_server():
952
+ self.server_thread = current_thread()
953
+ self.enter_wait_list()
954
+ server = plugin_jm_server.JmServer(base_dir, password, **kwargs)
955
+ # run方法会阻塞当前线程直到flask退出
956
+ server.run(**run)
957
+
958
+ # 对于debug模式,特殊处理
959
+ if run['debug'] is True:
960
+ run.setdefault('use_reloader', False)
961
+
962
+ # debug模式只能在主线程启动,判断当前线程是不是主线程
963
+ if current_thread() is not threading.main_thread():
964
+ # 不是主线程,return
965
+ return self.warning_wrong_usage_of_debug()
966
+ else:
967
+ self.running = True
968
+ # 是主线程,启动服务器
969
+ blocking_run_server()
970
+
971
+ else:
972
+ # 非debug模式,开新线程启动
973
+ threading.Thread(target=blocking_run_server, daemon=True).start()
974
+ atexit_register(self.wait_server_stop)
975
+ self.running = True
976
+
977
+ def warning_wrong_usage_of_debug(self):
978
+ self.log('注意!当配置debug=True时,请确保当前插件是在主线程中被调用。\n'
979
+ '因为如果本插件配置在 [after_album/after_photo] 这种时机调用,\n'
980
+ '会使得flask框架不在主线程debug运行,\n'
981
+ '导致报错(ValueError: signal only works in main thread of the main interpreter)。\n',
982
+ '【基于上述原因,当前线程非主线程,不启动服务器】'
983
+ 'warning'
984
+ )
985
+
986
+ def wait_server_stop(self, proactive=False):
987
+ st = self.server_thread
988
+ if (
989
+ st is None
990
+ or st == current_thread()
991
+ or not st.is_alive()
992
+ ):
993
+ return
994
+
995
+ if proactive:
996
+ msg = f'[{self.plugin_key}]的服务器线程仍运行中,可按下ctrl+c结束程序'
997
+ else:
998
+ msg = f'主线程执行完毕,但插件[{self.plugin_key}]的服务器线程仍运行中,可按下ctrl+c结束程序'
999
+
1000
+ self.log(msg, 'wait')
1001
+
1002
+ while st.is_alive():
1003
+ try:
1004
+ st.join(timeout=0.5)
1005
+ except KeyboardInterrupt:
1006
+ self.log('收到ctrl+c,结束程序', 'wait')
1007
+ return
1008
+
1009
+ def wait_until_finish(self):
1010
+ self.wait_server_stop(proactive=True)
1011
+
1012
+ @classmethod
1013
+ def build(cls, option: JmOption) -> 'JmOptionPlugin':
1014
+ """
1015
+ 单例模式
1016
+ """
1017
+ field_name = 'single_instance'
1018
+
1019
+ instance = getattr(cls, field_name, None)
1020
+ if instance is not None:
1021
+ return instance
1022
+
1023
+ with cls.single_instance_lock:
1024
+ instance = getattr(cls, field_name, None)
1025
+ if instance is not None:
1026
+ return instance
1027
+ instance = JmServerPlugin(option)
1028
+ setattr(cls, field_name, instance)
1029
+ return instance
1030
+
1031
+
1032
+ class SubscribeAlbumUpdatePlugin(JmOptionPlugin):
1033
+ plugin_key = 'subscribe_album_update'
1034
+
1035
+ def invoke(self,
1036
+ album_photo_dict=None,
1037
+ email_notify=None,
1038
+ download_if_has_update=True,
1039
+ auto_update_after_download=True,
1040
+ ) -> None:
1041
+ if album_photo_dict is None:
1042
+ return
1043
+
1044
+ album_photo_dict: Dict
1045
+ for album_id, photo_id in album_photo_dict.copy().items():
1046
+ # check update
1047
+ try:
1048
+ has_update, photo_new_list = self.check_photo_update(album_id, photo_id)
1049
+ except JmcomicException as e:
1050
+ self.log('Exception happened: ' + str(e), 'check_update.error')
1051
+ continue
1052
+
1053
+ if has_update is False:
1054
+ continue
1055
+
1056
+ self.log(f'album={album_id},发现新章节: {photo_new_list},准备开始下载')
1057
+
1058
+ # send email
1059
+ try:
1060
+ if email_notify:
1061
+ SendQQEmailPlugin.build(self.option).invoke(**email_notify)
1062
+ except PluginValidationException:
1063
+ # ignore
1064
+ pass
1065
+
1066
+ # download new photo
1067
+ if has_update and download_if_has_update:
1068
+ self.option.download_photo(photo_new_list)
1069
+
1070
+ if auto_update_after_download:
1071
+ album_photo_dict[album_id] = photo_new_list[-1]
1072
+ self.option.to_file()
1073
+
1074
+ def check_photo_update(self, album_id: str, photo_id: str):
1075
+ client = self.option.new_jm_client()
1076
+ album = client.get_album_detail(album_id)
1077
+
1078
+ photo_new_list = []
1079
+ is_new_photo = False
1080
+ sentinel = int(photo_id)
1081
+
1082
+ for photo in album:
1083
+ if is_new_photo:
1084
+ photo_new_list.append(photo.photo_id)
1085
+
1086
+ if int(photo.photo_id) == sentinel:
1087
+ is_new_photo = True
1088
+
1089
+ return len(photo_new_list) != 0, photo_new_list
1090
+
1091
+
1092
+ class SkipPhotoWithFewImagesPlugin(JmOptionPlugin):
1093
+ plugin_key = 'skip_photo_with_few_images'
1094
+
1095
+ def invoke(self,
1096
+ at_least_image_count: int,
1097
+ photo: Optional[JmPhotoDetail] = None,
1098
+ image: Optional[JmImageDetail] = None,
1099
+ album: Optional[JmAlbumDetail] = None,
1100
+ **kwargs
1101
+ ):
1102
+ self.try_mark_photo_skip_and_log(photo, at_least_image_count)
1103
+ if image is not None:
1104
+ self.try_mark_photo_skip_and_log(image.from_photo, at_least_image_count)
1105
+
1106
+ def try_mark_photo_skip_and_log(self, photo: JmPhotoDetail, at_least_image_count: int):
1107
+ if photo is None:
1108
+ return
1109
+
1110
+ if len(photo) >= at_least_image_count:
1111
+ return
1112
+
1113
+ self.log(f'跳过下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}]),'
1114
+ f'因为其图片数: {len(photo)} < {at_least_image_count} (at_least_image_count)')
1115
+ photo.skip = True
1116
+
1117
+ @classmethod
1118
+ @field_cache() # 单例
1119
+ def build(cls, option: JmOption) -> 'JmOptionPlugin':
1120
+ return super().build(option)
1121
+
1122
+
1123
+ class DeleteDuplicatedFilesPlugin(JmOptionPlugin):
1124
+ """
1125
+ https://github.com/hect0x7/JMComic-Crawler-Python/issues/244
1126
+ """
1127
+ plugin_key = 'delete_duplicated_files'
1128
+
1129
+ @classmethod
1130
+ def calculate_md5(cls, file_path):
1131
+ import hashlib
1132
+
1133
+ """计算文件的MD5哈希值"""
1134
+ hash_md5 = hashlib.md5()
1135
+ with open(file_path, "rb") as f:
1136
+ for chunk in iter(lambda: f.read(4096), b""):
1137
+ hash_md5.update(chunk)
1138
+ return hash_md5.hexdigest()
1139
+
1140
+ @classmethod
1141
+ def find_duplicate_files(cls, root_folder):
1142
+ """递归读取文件夹下所有文件并计算MD5出现次数"""
1143
+ import os
1144
+ from collections import defaultdict
1145
+ md5_dict = defaultdict(list)
1146
+
1147
+ for root, _, files in os.walk(root_folder):
1148
+ for file in files:
1149
+ file_path = os.path.join(root, file)
1150
+ file_md5 = cls.calculate_md5(file_path)
1151
+ md5_dict[file_md5].append(file_path)
1152
+
1153
+ return md5_dict
1154
+
1155
+ def invoke(self,
1156
+ limit,
1157
+ album=None,
1158
+ downloader=None,
1159
+ delete_original_file=True,
1160
+ **kwargs,
1161
+ ) -> None:
1162
+ if album is None:
1163
+ return
1164
+
1165
+ self.delete_original_file = delete_original_file
1166
+ # 获取到下载本子所在根目录
1167
+ root_folder = self.option.dir_rule.decide_album_root_dir(album)
1168
+ self.find_duplicated_files_and_delete(limit, root_folder, album)
1169
+
1170
+ def find_duplicated_files_and_delete(self, limit: int, root_folder: str, album: Optional[JmAlbumDetail] = None):
1171
+ md5_dict = self.find_duplicate_files(root_folder)
1172
+ # 打印MD5出现次数大于等于limit的文件
1173
+ for md5, paths in md5_dict.items():
1174
+ if len(paths) >= limit:
1175
+ prefix = '' if album is None else f'({album.album_id}) '
1176
+ message = [prefix + f'MD5: {md5} 出现次数: {len(paths)}'] + \
1177
+ [f' {path}' for path in paths]
1178
+ self.log('\n'.join(message))
1179
+ self.execute_deletion(paths)
1180
+
1181
+
1182
+ class ReplacePathStringPlugin(JmOptionPlugin):
1183
+ plugin_key = 'replace_path_string'
1184
+
1185
+ def invoke(self,
1186
+ replace: Dict[str, str],
1187
+ ):
1188
+ if not replace:
1189
+ return
1190
+
1191
+ old_decide_dir = self.option.decide_image_save_dir
1192
+
1193
+ def new_decide_dir(photo, ensure_exists=True) -> str:
1194
+ original_path: str = old_decide_dir(photo, False)
1195
+ for k, v in replace.items():
1196
+ original_path = original_path.replace(k, v)
1197
+
1198
+ if ensure_exists:
1199
+ JmcomicText.try_mkdir(original_path)
1200
+
1201
+ return original_path
1202
+
1203
+ self.option.decide_image_save_dir = new_decide_dir