jmcomic 2.5.8__tar.gz → 2.5.10__tar.gz

{jmcomic-2.5.8/src/jmcomic.egg-info → jmcomic-2.5.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jmcomic
-Version: 2.5.8
+Version: 2.5.10
 Summary: Python API For JMComic (禁漫天堂)
 Home-page: https://github.com/hect0x7/JMComic-Crawler-Python
 Author: hect0x7
@@ -20,8 +20,8 @@ Classifier: Operating System :: Microsoft :: Windows
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: commonX>=0.6.4
 Requires-Dist: curl_cffi
+Requires-Dist: commonX
 Requires-Dist: PyYAML
 Requires-Dist: Pillow
 Requires-Dist: pycryptodome

{jmcomic-2.5.8 → jmcomic-2.5.10}/setup.py

@@ -27,8 +27,8 @@ setup(
     package_dir={"": "src"},
     python_requires=">=3.7",
     install_requires=[
-        'commonX>=0.6.4',
         'curl_cffi',
+        'commonX',
         'PyYAML',
         'Pillow',
         'pycryptodome',

{jmcomic-2.5.8 → jmcomic-2.5.10}/src/jmcomic/__init__.py

@@ -2,7 +2,7 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- option <--- downloader
 
-__version__ = '2.5.8'
+__version__ = '2.5.10'
 
 from .api import *
 from .jm_plugin import *

{jmcomic-2.5.8 → jmcomic-2.5.10}/src/jmcomic/jm_client_impl.py

@@ -224,7 +224,7 @@ class AbstractJmClient(
 
     # noinspection PyMethodMayBeStatic
     def decode(self, url: str):
-        if not JmModuleConfig.flag_decode_url_when_logging or '/search/' not in url:
+        if not JmModuleConfig.FLAG_DECODE_URL_WHEN_LOGGING or '/search/' not in url:
             return url
 
         from urllib.parse import unquote
@@ -767,7 +767,7 @@ class JmApiClient(AbstractJmClient):
         # 检查禁漫最新的版本号
         setting_ver = str(resp.model_data.version)
         # 禁漫接口的版本 > jmcomic库内置版本
-        if setting_ver > JmMagicConstants.APP_VERSION and JmModuleConfig.flag_use_version_newer_if_behind:
+        if setting_ver > JmMagicConstants.APP_VERSION and JmModuleConfig.FLAG_USE_VERSION_NEWER_IF_BEHIND:
             jm_log('api.setting', f'change APP_VERSION from [{JmMagicConstants.APP_VERSION}] to [{setting_ver}]')
             JmMagicConstants.APP_VERSION = setting_ver
 
@@ -883,7 +883,7 @@ class JmApiClient(AbstractJmClient):
             ts = time_stamp()
             token, tokenparam = JmCryptoTool.token_and_tokenparam(ts, secret=JmMagicConstants.APP_TOKEN_SECRET_2)
 
-        elif JmModuleConfig.flag_use_fix_timestamp:
+        elif JmModuleConfig.FLAG_USE_FIX_TIMESTAMP:
             ts, token, tokenparam = JmModuleConfig.get_fix_ts_token_tokenparam()
 
         else:
@@ -954,7 +954,7 @@ class JmApiClient(AbstractJmClient):
 
     def after_init(self):
         # 保证拥有cookies，因为移动端要求必须携带cookies，否则会直接跳转同一本子【禁漫娘】
-        if JmModuleConfig.flag_api_client_require_cookies:
+        if JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES:
             self.ensure_have_cookies()
 
     client_init_cookies_lock = Lock()

{jmcomic-2.5.8 → jmcomic-2.5.10}/src/jmcomic/jm_config.py

@@ -146,18 +146,18 @@ class JmModuleConfig:
     REGISTRY_EXCEPTION_LISTENER = {}
 
     # 执行log的函数
-    executor_log = default_jm_logging
+    EXECUTOR_LOG = default_jm_logging
 
     # 使用固定时间戳
-    flag_use_fix_timestamp = True
+    FLAG_USE_FIX_TIMESTAMP = True
     # 移动端Client初始化cookies
-    flag_api_client_require_cookies = True
+    FLAG_API_CLIENT_REQUIRE_COOKIES = True
     # log开关标记
-    flag_enable_jm_log = True
+    FLAG_ENABLE_JM_LOG = True
     # log时解码url
-    flag_decode_url_when_logging = True
+    FLAG_DECODE_URL_WHEN_LOGGING = True
     # 当内置的版本号落后时，使用最新的禁漫app版本号
-    flag_use_version_newer_if_behind = True
+    FLAG_USE_VERSION_NEWER_IF_BEHIND = True
 
     # 关联dir_rule的自定义字段与对应的处理函数
     # 例如:
@@ -165,6 +165,10 @@ class JmModuleConfig:
     AFIELD_ADVICE = dict()
     PFIELD_ADVICE = dict()
 
+    # 当发生 oserror: [Errno 36] File name too long 时，
+    # 把文件名限制在指定个字符以内
+    VAR_FILE_NAME_LENGTH_LIMIT = 100
+
     @classmethod
     def downloader_class(cls):
         if cls.CLASS_DOWNLOADER is not None:
@@ -319,12 +323,12 @@ class JmModuleConfig:
     # noinspection PyUnusedLocal
     @classmethod
     def jm_log(cls, topic: str, msg: str):
-        if cls.flag_enable_jm_log is True:
-            cls.executor_log(topic, msg)
+        if cls.FLAG_ENABLE_JM_LOG is True:
+            cls.EXECUTOR_LOG(topic, msg)
 
     @classmethod
     def disable_jm_log(cls):
-        cls.flag_enable_jm_log = False
+        cls.FLAG_ENABLE_JM_LOG = False
 
     @classmethod
     def new_postman(cls, session=False, **kwargs):
@@ -347,7 +351,7 @@ class JmModuleConfig:
     DEFAULT_CLIENT_CACHE = None  # 默认关闭Client缓存。缓存的配置详见 CacheRegistry
     DEFAULT_PROXIES = ProxyBuilder.system_proxy()  # 默认使用系统代理
 
-    default_option_dict: dict = {
+    DEFAULT_OPTION_DICT: dict = {
         'log': None,
         'dir_rule': {'rule': 'Bd_Pname', 'base_dir': None},
         'download': {
@@ -387,11 +391,11 @@ class JmModuleConfig:
         """
         from copy import deepcopy
 
-        option_dict = deepcopy(cls.default_option_dict)
+        option_dict = deepcopy(cls.DEFAULT_OPTION_DICT)
 
         # log
         if option_dict['log'] is None:
-            option_dict['log'] = cls.flag_enable_jm_log
+            option_dict['log'] = cls.FLAG_ENABLE_JM_LOG
 
         # dir_rule.base_dir
         dir_rule = option_dict['dir_rule']

{jmcomic-2.5.8 → jmcomic-2.5.10}/src/jmcomic/jm_downloader.py

@@ -29,7 +29,7 @@ class DownloadCallback:
                f'章节下载完成: [{photo.id}] ({photo.album_id}[{photo.index}/{len(photo.from_album)}])')
 
     def before_image(self, image: JmImageDetail, img_save_path):
-        if image.is_exists:
+        if image.exists:
             jm_log('image.before',
                    f'图片已存在: {image.tag} ← [{img_save_path}]'
                    )
@@ -63,6 +63,8 @@ class JmDownloader(DownloadCallback):
 
     def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
         self.before_album(album)
+        if album.skip:
+            return
         self.execute_by_condition(
             iter_objs=album,
             apply=lambda photo: self.download_by_photo_detail(photo, client),
@@ -80,6 +82,8 @@ class JmDownloader(DownloadCallback):
         client.check_photo(photo)
 
         self.before_photo(photo)
+        if photo.skip:
+            return
         self.execute_by_condition(
             iter_objs=photo,
             apply=lambda image: self.download_by_image_detail(image, client),
@@ -91,16 +95,19 @@ class JmDownloader(DownloadCallback):
         img_save_path = self.option.decide_image_filepath(image)
 
         image.save_path = img_save_path
-        image.is_exists = file_exists(img_save_path)
+        image.exists = file_exists(img_save_path)
 
         self.before_image(image, img_save_path)
 
+        if image.skip:
+            return
+
         # let option decide use_cache and decode_image
         use_cache = self.option.decide_download_cache(image)
         decode_image = self.option.decide_download_image_decode(image)
 
         # skip download
-        if use_cache is True and image.is_exists:
+        if use_cache is True and image.exists:
             return
 
         e = None

{jmcomic-2.5.8 → jmcomic-2.5.10}/src/jmcomic/jm_entity.py

@@ -3,6 +3,14 @@ from common import *
 from .jm_config import *
 
 
+class Downloadable:
+
+    def __init__(self):
+        self.save_path: str = ''
+        self.exists: bool = False
+        self.skip = False
+
+
 class JmBaseEntity:
 
     def to_file(self, filepath):
@@ -117,7 +125,7 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
     def __str__(self):
         return f'{self.__class__.__name__}' \
                '{' \
-               f'{self.id}: {self.title}'\
+               f'{self.id}: {self.title}' \
                '}'
 
     @classmethod
@@ -156,7 +164,7 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
         return getattr(detail, ref)
 
 
-class JmImageDetail(JmBaseEntity):
+class JmImageDetail(JmBaseEntity, Downloadable):
 
     def __init__(self,
                  aid,
@@ -167,7 +175,8 @@ class JmImageDetail(JmBaseEntity):
                  from_photo=None,
                  query_params=None,
                  index=-1,
-                 ) -> None:
+                 ):
+        super().__init__()
         if scramble_id is None or (isinstance(scramble_id, str) and scramble_id == ''):
             from .jm_toolkit import ExceptionTool
             ExceptionTool.raises(f'图片的scramble_id不能为空')
@@ -182,10 +191,6 @@ class JmImageDetail(JmBaseEntity):
         self.query_params: Optional[str] = query_params
         self.index = index  # 从1开始
 
-        # temp fields, in order to simplify passing parameter
-        self.save_path: str = ''
-        self.is_exists: bool = False
-
     @property
     def filename_without_suffix(self):
         return self.img_file_name
@@ -252,7 +257,7 @@ class JmImageDetail(JmBaseEntity):
         return True
 
 
-class JmPhotoDetail(DetailEntity):
+class JmPhotoDetail(DetailEntity, Downloadable):
 
     def __init__(self,
                  photo_id,
@@ -267,6 +272,7 @@ class JmPhotoDetail(DetailEntity):
                  author=None,
                  from_album=None,
                  ):
+        super().__init__()
         self.photo_id: str = str(photo_id)
         self.scramble_id: str = str(scramble_id)
         self.name: str = str(name).strip()
@@ -411,7 +417,7 @@ class JmPhotoDetail(DetailEntity):
         return True
 
 
-class JmAlbumDetail(DetailEntity):
+class JmAlbumDetail(DetailEntity, Downloadable):
 
     def __init__(self,
                  album_id,
@@ -430,6 +436,7 @@ class JmAlbumDetail(DetailEntity):
                  tags,
                  related_list=None,
                  ):
+        super().__init__()
         self.album_id: str = str(album_id)
         self.scramble_id: str = str(scramble_id)
         self.name: str = name

{jmcomic-2.5.8 → jmcomic-2.5.10}/src/jmcomic/jm_exception.py

@@ -72,13 +72,6 @@ class ExceptionTool:
     CONTEXT_KEY_RE_PATTERN = 'pattern'
     CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id'
 
-    # 兼容旧版本
-
-    EXTRA_KEY_RESP = 'resp'
-    EXTRA_KEY_HTML = 'html'
-    EXTRA_KEY_RE_PATTERN = 'pattern'
-    EXTRA_KEY_MISSING_JM_ID = 'missing_jm_id'
-
     @classmethod
     def raises(cls,
                msg: str,
@@ -144,7 +137,8 @@ class ExceptionTool:
         :param resp: 响应对象
         :param jmid: 禁漫本子/章节id
         """
-        url = resp.url
+        from .jm_toolkit import JmcomicText
+        url = JmcomicText.format_album_url(jmid)
 
         req_type = "本子" if "album" in url else "章节"
         cls.raises(

{jmcomic-2.5.8 → jmcomic-2.5.10}/src/jmcomic/jm_option.py

@@ -236,28 +236,6 @@ class JmOption:
     def decide_photo_batch_count(self, album: JmAlbumDetail):
         return self.download.threading.photo
 
-    def decide_album_dir(self, album: JmAlbumDetail) -> str:
-        """
-        该方法目前仅在 plugin-zip 中使用，不建议外部调用
-        """
-        dir_layer = []
-        dir_rule = self.dir_rule
-        for rule in dir_rule.rule_dsl.split('_'):
-            if rule == 'Bd':
-                dir_layer.append(dir_rule.base_dir)
-                continue
-
-            if rule[0] == 'A':
-                name = dir_rule.apply_rule_directly(album, None, rule)
-                dir_layer.append(name)
-
-            if rule[0] == 'P':
-                break
-
-        from os.path import join
-        # noinspection PyTypeChecker
-        return join(*dir_layer)
-
     # noinspection PyMethodMayBeStatic
     def decide_image_filename(self, image: JmImageDetail) -> str:
         """
@@ -285,7 +263,15 @@ class JmOption:
         )
 
         if ensure_exists:
-            mkdir_if_not_exists(save_dir)
+            try:
+                mkdir_if_not_exists(save_dir)
+            except OSError as e:
+                if e.errno == 36:
+                    # 目录名过长
+                    limit = JmModuleConfig.VAR_FILE_NAME_LENGTH_LIMIT
+                    jm_log('error', f'目录名过长，无法创建目录，强制缩短到{limit}个字符并重试')
+                    save_dir = save_dir[0:limit]
+                    mkdir_if_not_exists(save_dir)
 
         return save_dir
 
@@ -359,7 +345,7 @@ class JmOption:
     def deconstruct(self) -> Dict:
         return {
             'version': JmModuleConfig.JM_OPTION_VER,
-            'log': JmModuleConfig.flag_enable_jm_log,
+            'log': JmModuleConfig.FLAG_ENABLE_JM_LOG,
             'dir_rule': {
                 'rule': self.dir_rule.rule_dsl,
                 'base_dir': self.dir_rule.base_dir,

{jmcomic-2.5.8 → jmcomic-2.5.10}/src/jmcomic/jm_plugin.py

@@ -302,27 +302,19 @@ class ZipPlugin(JmOptionPlugin):
 
         if level == 'album':
             zip_path = self.get_zip_path(album, None, filename_rule, suffix, zip_dir)
-            dir_path = self.zip_album(album, photo_dict, zip_path)
-            if dir_path is not None:
-                # 要删除这个album文件夹
-                dir_zip_dict[dir_path] = zip_path
-                # 也要删除album下的photo文件夹
-                for d in files_of_dir(dir_path):
-                    dir_zip_dict[d] = None
+            self.zip_album(album, photo_dict, zip_path, dir_zip_dict)
 
         elif level == 'photo':
             for photo, image_list in photo_dict.items():
                 zip_path = self.get_zip_path(None, photo, filename_rule, suffix, zip_dir)
-                dir_path = self.zip_photo(photo, image_list, zip_path)
-                if dir_path is not None:
-                    dir_zip_dict[dir_path] = zip_path
+                self.zip_photo(photo, image_list, zip_path, dir_zip_dict)
 
         else:
             ExceptionTool.raises(f'Not Implemented Zip Level: {level}')
 
         self.after_zip(dir_zip_dict)
 
-    def zip_photo(self, photo, image_list: list, zip_path: str) -> Optional[str]:
+    def zip_photo(self, photo, image_list: list, zip_path: str, dir_zip_dict) -> Optional[str]:
         """
         压缩photo文件夹
         :returns: photo文件夹路径
@@ -333,50 +325,58 @@ class ZipPlugin(JmOptionPlugin):
 
         all_filepath = set(map(lambda t: self.unified_path(t[0]), image_list))
 
-        return self.do_zip(photo_dir,
-                           zip_path,
-                           all_filepath,
-                           f'压缩章节[{photo.photo_id}]成功 → {zip_path}',
-                           )
+        if len(all_filepath) == 0:
+            self.log('无下载文件，无需压缩', 'skip')
+            return None
+
+        from common import backup_dir_to_zip
+        backup_dir_to_zip(
+            photo_dir,
+            zip_path,
+            acceptor=lambda f: os.path.isdir(f) or self.unified_path(f) in all_filepath
+        ).close()
+
+        self.log(f'压缩章节[{photo.photo_id}]成功 → {zip_path}', 'finish')
+        dir_zip_dict[self.unified_path(photo_dir)] = zip_path
 
     @staticmethod
     def unified_path(f):
         return fix_filepath(f, os.path.isdir(f))
 
-    def zip_album(self, album, photo_dict: dict, zip_path) -> Optional[str]:
+    def zip_album(self, album, photo_dict: dict, zip_path, dir_zip_dict) -> Optional[str]:
         """
         压缩album文件夹
         :returns: album文件夹路径
         """
-        all_filepath: Set[str] = set()
-
-        def addpath(f):
-            all_filepath.update(set(f))
-
-        album_dir = self.option.decide_album_dir(album)
-        # addpath(self.option.decide_image_save_dir(photo) for photo in photo_dict.keys())
-        addpath(path for ls in photo_dict.values() for path, _ in ls)
 
-        return self.do_zip(album_dir,
-                           zip_path,
-                           all_filepath,
-                           msg=f'压缩本子[{album.album_id}]成功 → {zip_path}',
-                           )
+        # 所有下载了的图片文件的路径
+        all_filepath: Set[str] = set(path for ls in photo_dict.values() for path, _ in ls)
 
-    def do_zip(self, source_dir, zip_path, all_filepath, msg):
         if len(all_filepath) == 0:
             self.log('无下载文件，无需压缩', 'skip')
-            return None
+            return
 
+        # 该本子的所有章节的图片所在文件夹
+        photo_dir_list = [self.option.decide_image_save_dir(photo) for photo in photo_dict.keys()]
+
+        # 压缩文件对象
         from common import backup_dir_to_zip
-        backup_dir_to_zip(
-            source_dir,
-            zip_path,
-            acceptor=lambda f: os.path.isdir(f) or self.unified_path(f) in all_filepath
-        ).close()
+        import zipfile
+        zfile = zipfile.ZipFile(zip_path, 'w')
+
+        for photo_dir in photo_dir_list:
+            photo_dir = self.unified_path(photo_dir)
+            backup_dir_to_zip(
+                photo_dir,
+                zip_path,
+                zfile=zfile,
+                prefix=os.path.basename(photo_dir.rstrip('/')),
+                acceptor=lambda f: os.path.isdir(f) or self.unified_path(f) in all_filepath
+            )
+            dir_zip_dict[photo_dir] = zip_path
 
-        self.log(msg, 'finish')
-        return self.unified_path(source_dir)
+        zfile.close()
+        self.log(f'压缩本子[{album.album_id}]成功 → {zip_path}', 'finish')
 
     def after_zip(self, dir_zip_dict: Dict[str, Optional[str]]):
         # 删除所有原文件
@@ -445,9 +445,7 @@ class ImageSuffixFilterPlugin(JmOptionPlugin):
             if image.img_file_suffix not in allowed_suffix_set:
                 self.log(f'跳过下载图片: {image.tag}，'
                          f'因为其后缀\'{image.img_file_suffix}\'不在允许的后缀集合{allowed_suffix_set}内')
-                # hook is_exists True to skip download
-                image.is_exists = True
-                return True
+                image.skip = True
 
             # let option decide
             return option_decide_cache(image)
@@ -484,7 +482,7 @@ class LogTopicFilterPlugin(JmOptionPlugin):
         if whitelist is not None:
             whitelist = set(whitelist)
 
-        old_jm_log = JmModuleConfig.executor_log
+        old_jm_log = JmModuleConfig.EXECUTOR_LOG
 
         def new_jm_log(topic, msg):
             if whitelist is not None and topic not in whitelist:
@@ -492,7 +490,7 @@ class LogTopicFilterPlugin(JmOptionPlugin):
 
             old_jm_log(topic, msg)
 
-        JmModuleConfig.executor_log = new_jm_log
+        JmModuleConfig.EXECUTOR_LOG = new_jm_log
 
 
 class AutoSetBrowserCookiesPlugin(JmOptionPlugin):
@@ -963,3 +961,34 @@ class SubscribeAlbumUpdatePlugin(JmOptionPlugin):
                 is_new_photo = True
 
         return len(photo_new_list) != 0, photo_new_list
+
+
+class SkipPhotoWithFewImagesPlugin(JmOptionPlugin):
+    plugin_key = 'skip_photo_with_few_images'
+
+    def invoke(self,
+               at_least_image_count: int,
+               photo: Optional[JmPhotoDetail] = None,
+               image: Optional[JmImageDetail] = None,
+               album: Optional[JmAlbumDetail] = None,
+               **kwargs
+               ):
+        self.try_mark_photo_skip_and_log(photo, at_least_image_count)
+        if image is not None:
+            self.try_mark_photo_skip_and_log(image.from_photo, at_least_image_count)
+
+    def try_mark_photo_skip_and_log(self, photo: JmPhotoDetail, at_least_image_count: int):
+        if photo is None:
+            return
+
+        if len(photo) >= at_least_image_count:
+            return
+
+        self.log(f'跳过下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}])，'
+                 f'因为其图片数: {len(photo)} < {at_least_image_count} (at_least_image_count)')
+        photo.skip = True
+
+    @classmethod
+    @field_cache()  # 单例
+    def build(cls, option: JmOption) -> 'JmOptionPlugin':
+        return super().build(option)

{jmcomic-2.5.8 → jmcomic-2.5.10/src/jmcomic.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jmcomic
-Version: 2.5.8
+Version: 2.5.10
 Summary: Python API For JMComic (禁漫天堂)
 Home-page: https://github.com/hect0x7/JMComic-Crawler-Python
 Author: hect0x7
@@ -20,8 +20,8 @@ Classifier: Operating System :: Microsoft :: Windows
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: commonX>=0.6.4
 Requires-Dist: curl_cffi
+Requires-Dist: commonX
 Requires-Dist: PyYAML
 Requires-Dist: Pillow
 Requires-Dist: pycryptodome

{jmcomic-2.5.8 → jmcomic-2.5.10}/src/jmcomic.egg-info/requires.txt

@@ -1,5 +1,5 @@
-commonX>=0.6.4
 curl_cffi
+commonX
 PyYAML
 Pillow
 pycryptodome