jmcomic 2.6.6__py3-none-any.whl → 2.6.7__py3-none-any.whl

jmcomic/__init__.py

@@ -2,7 +2,7 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- option <--- downloader
 
-__version__ = '2.6.6'
+__version__ = '2.6.7'
 
 from .api import *
 from .jm_plugin import *

jmcomic/jm_client_impl.py

@@ -895,7 +895,7 @@ class JmApiClient(AbstractJmClient):
         检查返回数据中的status字段是否为ok
         """
         data = resp.model_data
-        if data.status == 'ok':
+        if data.status != 'ok':
             ExceptionTool.raises_resp(data.msg, resp)
 
     def req_api(self, url, get=True, require_success=True, **kwargs) -> JmApiResp:
@@ -995,7 +995,7 @@ class JmApiClient(AbstractJmClient):
                 # 找到第一个有效字符
                 ExceptionTool.require_true(
                     char == '{',
-                    f'请求不是json格式，强制重试！响应文本: [{resp.text}]'
+                    f'请求不是json格式，强制重试！响应文本: [{JmcomicText.limit_text(text, 200)}]'
                 )
                 return resp
 

jmcomic/jm_client_interface.py

@@ -101,6 +101,14 @@ class JmApiResp(JmJsonResp):
         super().__init__(resp)
         self.ts = ts
 
+    # 重写json()方法，可以忽略一些非json格式的脏数据
+    @field_cache()
+    def json(self) -> Dict:
+        try:
+            return JmcomicText.try_parse_json_object(self.resp.text)
+        except Exception as e:
+            ExceptionTool.raises_resp(f'json解析失败: {e}', self, JsonResolveFailException)
+
     @property
     def is_success(self) -> bool:
         return super().is_success and self.json()['code'] == 200

jmcomic/jm_option.py

@@ -419,7 +419,7 @@ class JmOption:
         if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient):
             raise NotImplementedError(clazz)
 
-        client: AbstractJmClient = clazz(
+        client: JmcomicClient = clazz(
             postman=postman,
             domain_list=decide_domain_list(),
             retry_times=retry_times,

jmcomic/jm_plugin.py

@@ -1221,23 +1221,88 @@ class ReplacePathStringPlugin(JmOptionPlugin):
 class AdvancedRetryPlugin(JmOptionPlugin):
     plugin_key = 'advanced-retry'
 
+    def __init__(self, option: JmOption):
+        super().__init__(option)
+        self.retry_config = None
+
     def invoke(self,
                retry_config,
                **kwargs):
+        self.require_param(isinstance(retry_config, dict), '必须配置retry_config为dict')
+        self.retry_config = retry_config
+
         new_jm_client: Callable = self.option.new_jm_client
 
         def hook_new_jm_client(*args, **kwargs):
-            client: AbstractJmClient = new_jm_client(*args, **kwargs)
+            client: JmcomicClient = new_jm_client(*args, **kwargs)
             client.domain_retry_strategy = self.request_with_retry
+            client.domain_req_failed_counter = {}
+            from threading import Lock
+            client.domain_counter_lock = Lock()
             return client
 
         self.option.new_jm_client = hook_new_jm_client
 
     def request_with_retry(self,
-                           client,
-                           request,
-                           url,
-                           is_image,
+                           client: AbstractJmClient,
+                           request: Callable,
+                           url: str,
+                           is_image: bool,
                            **kwargs,
                            ):
-        pass
+        """
+        实现如下域名重试机制：
+        - 对域名列表轮询请求，配置：retry_rounds
+        - 限制单个域名最大失败次数，配置：retry_domain_max_times
+        - 轮询域名列表前，根据历史失败次数对域名列表排序，失败多的后置
+        """
+
+        def do_request(domain):
+            url_to_use = url
+            if url_to_use.startswith('/'):
+                # path → url
+                url_to_use = client.of_api_url(url, domain)
+                client.update_request_with_specify_domain(kwargs, domain, is_image)
+                jm_log(client.log_topic(), client.decode(url_to_use))
+            elif is_image:
+                # 图片url
+                client.update_request_with_specify_domain(kwargs, None, is_image)
+
+            resp = request(url_to_use, **kwargs)
+            resp = client.raise_if_resp_should_retry(resp, is_image)
+            return resp
+
+        retry_domain_max_times: int = self.retry_config['retry_domain_max_times']
+        retry_rounds: int = self.retry_config['retry_rounds']
+        for rindex in range(retry_rounds):
+            domain_list = self.get_sorted_domain(client, retry_domain_max_times)
+            for i, domain in enumerate(domain_list):
+                if self.failed_count(client, domain) >= retry_domain_max_times:
+                    continue
+
+                try:
+                    return do_request(domain)
+                except Exception as e:
+                    from common import traceback_print_exec
+                    traceback_print_exec()
+                    jm_log('req.error', str(e))
+                    self.update_failed_count(client, domain)
+
+        return client.fallback(request, url, 0, 0, is_image, **kwargs)
+
+    def get_sorted_domain(self, client: JmcomicClient, times):
+        domain_list = client.get_domain_list()
+        return sorted(
+            filter(lambda d: self.failed_count(client, d) < times, domain_list),
+            key=lambda d: self.failed_count(client, d)
+        )
+
+    # noinspection PyUnresolvedReferences
+    def update_failed_count(self, client: AbstractJmClient, domain: str):
+        with client.domain_counter_lock:
+            client.domain_req_failed_counter[domain] = self.failed_count(client, domain) + 1
+
+    @staticmethod
+    def failed_count(client: JmcomicClient, domain: str) -> int:
+        # noinspection PyUnresolvedReferences
+        return client.domain_req_failed_counter.get(domain, 0)

jmcomic/jm_toolkit.py

@@ -61,6 +61,8 @@ class JmcomicText:
 
     # 提取接口返回值信息
     pattern_ajax_favorite_msg = compile(r'</button>(.*?)</div>')
+    # 提取api接口返回值里的json，防止返回值里有无关日志导致json解析报错
+    pattern_api_response_json_object = compile(r'\{[\s\S]*?}')
 
     @classmethod
     def parse_to_jm_domain(cls, text: str):
@@ -344,6 +346,28 @@ class JmcomicText:
             raise e
         return save_dir
 
+    # noinspection PyTypeChecker
+    @classmethod
+    def try_parse_json_object(cls, resp_text: str) -> dict:
+        import json
+        text = resp_text.strip()
+        if text.startswith('{') and text.endswith('}'):
+            # fast case
+            return json.loads(text)
+
+        for match in cls.pattern_api_response_json_object.finditer(text):
+            try:
+                return json.loads(match.group(0))
+            except Exception as e:
+                jm_log('parse_json_object.error', e)
+
+        raise AssertionError(f'未解析出json数据: {cls.limit_text(resp_text, 200)}')
+
+    @classmethod
+    def limit_text(cls, text: str, limit: int) -> str:
+        length = len(text)
+        return text if length <= limit else (text[:limit] + f'...({length - limit}')
+
 
 # 支持dsl: #{???} -> os.getenv(???)
 JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
@@ -450,10 +474,7 @@ class JmPageTool:
             # 这里不作解析，因为没什么用...
             tags = cls.pattern_html_search_tags.findall(tag_text)
             content.append((
-                album_id, {
-                    'name': title,  # 改成name是为了兼容 parse_api_resp_to_page
-                    'tags': tags
-                }
+                album_id, dict(name=title, tags=tags)  # 改成name是为了兼容 parse_api_resp_to_page
             ))
 
         return JmSearchPage(content, total)
@@ -468,10 +489,7 @@ class JmPageTool:
         for (album_id, title, tag_text) in album_info_list:
             tags = cls.pattern_html_search_tags.findall(tag_text)
             content.append((
-                album_id, {
-                    'name': title,  # 改成name是为了兼容 parse_api_resp_to_page
-                    'tags': tags
-                }
+                album_id, dict(name=title, tags=tags)  # 改成name是为了兼容 parse_api_resp_to_page
             ))
 
         return JmSearchPage(content, total)

{jmcomic-2.6.6.dist-info → jmcomic-2.6.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jmcomic
-Version: 2.6.6
+Version: 2.6.7
 Summary: Python API For JMComic (禁漫天堂)
 Home-page: https://github.com/hect0x7/JMComic-Crawler-Python
 Author: hect0x7

jmcomic-2.6.7.dist-info/RECORD

@@ -0,0 +1,18 @@
+jmcomic/__init__.py,sha256=Rg9DJ3YptC-BImHcbNB4cqM6-ZZMMtPGy2hw3-G7hZY,902
+jmcomic/api.py,sha256=ZduhXDmh4lg1dkXHs7UTAaPpYNO7kcdPCDH5JJT9KSI,4253
+jmcomic/cl.py,sha256=PBSh0JndNFZw3B7WJPj5Y8SeFdKzHE00jIwYo9An-K0,3475
+jmcomic/jm_client_impl.py,sha256=sDrhqlO6zHAR0vlGFoKXmQ85l89BwksfG8BHGk7eXJ4,42165
+jmcomic/jm_client_interface.py,sha256=4TDz-V8XLWPPxogLI-xiiHug4n6iir6UNeiKx9-KTXM,19410
+jmcomic/jm_config.py,sha256=8GDETnzQObZabd441Boc--Mb--J2kr7HLxxViKB0DvI,17213
+jmcomic/jm_downloader.py,sha256=azYkIN-1OuEivZ1TGGDaYo1FkcO5KatSb0g4GVdKSPY,11041
+jmcomic/jm_entity.py,sha256=ZyXVy3mOl9qkxlw6hyshVS3RR-yKfO5BkJHtyBAoKQE,20053
+jmcomic/jm_exception.py,sha256=x3KGMLlQS2zi1GX7z5G58zJN2EwLkI4mAURkxZYjEvA,5055
+jmcomic/jm_option.py,sha256=T__PXJgzITdjyr2PMdk98jYMIKkUVGUA30p5lsMOTvQ,21244
+jmcomic/jm_plugin.py,sha256=OekMjPVS25PMLcHV0r434BBvy0vbszlNiU5wODCZ3js,44524
+jmcomic/jm_toolkit.py,sha256=9vwk4Q2gfh_04BhJxEoSsZcDT6ICh-wsN70WR4LSD9I,30511
+jmcomic-2.6.7.dist-info/licenses/LICENSE,sha256=kz4coTxZxuGxisK3W00tjK57Zh3RcMGq-EnbXrK7-xA,1064
+jmcomic-2.6.7.dist-info/METADATA,sha256=cuqBqo5Bp2GHItulju3coNt5dU9_UMtY1vcN0G0m0jk,9572
+jmcomic-2.6.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+jmcomic-2.6.7.dist-info/entry_points.txt,sha256=tRbQltaGSBjejI0c9jYt-4SXQMd5nSDHcMvHmuTy4ow,44
+jmcomic-2.6.7.dist-info/top_level.txt,sha256=puvVMFYJqIbd6NOTMEvOyugMTT8woBfSQyxEBan3zY4,8
+jmcomic-2.6.7.dist-info/RECORD,,

jmcomic-2.6.6.dist-info/RECORD

@@ -1,18 +0,0 @@
-jmcomic/__init__.py,sha256=jl9p808Fwi2NHktr1RmupkJuZ4GbRqTzExT2CouapZY,902
-jmcomic/api.py,sha256=ZduhXDmh4lg1dkXHs7UTAaPpYNO7kcdPCDH5JJT9KSI,4253
-jmcomic/cl.py,sha256=PBSh0JndNFZw3B7WJPj5Y8SeFdKzHE00jIwYo9An-K0,3475
-jmcomic/jm_client_impl.py,sha256=atj557YHa0LDOezL9TSZrRA5lV9fs7CJzFLD1_MrONU,42141
-jmcomic/jm_client_interface.py,sha256=Bld80cYwBwIqcMSJK09g5q8U_UX3IWnQJxuRJQwmIOk,19082
-jmcomic/jm_config.py,sha256=8GDETnzQObZabd441Boc--Mb--J2kr7HLxxViKB0DvI,17213
-jmcomic/jm_downloader.py,sha256=azYkIN-1OuEivZ1TGGDaYo1FkcO5KatSb0g4GVdKSPY,11041
-jmcomic/jm_entity.py,sha256=ZyXVy3mOl9qkxlw6hyshVS3RR-yKfO5BkJHtyBAoKQE,20053
-jmcomic/jm_exception.py,sha256=x3KGMLlQS2zi1GX7z5G58zJN2EwLkI4mAURkxZYjEvA,5055
-jmcomic/jm_option.py,sha256=kErkn9-tXYp4h4lDQvc8jDYG1vqOMLIdIb3fbHvSsbA,21247
-jmcomic/jm_plugin.py,sha256=0SnL6bc5kUeDoculz_YxlOdxFFg1B0EkAcDV1K0GIAk,41633
-jmcomic/jm_toolkit.py,sha256=SIYCRZ6P4SPbAQA03U5z2DNiDldY7hWsAvKTdEzyK9U,29683
-jmcomic-2.6.6.dist-info/licenses/LICENSE,sha256=kz4coTxZxuGxisK3W00tjK57Zh3RcMGq-EnbXrK7-xA,1064
-jmcomic-2.6.6.dist-info/METADATA,sha256=bTugkFXmdS6nzfEk7hL6jCcweUBg0ntz3ksRaHJtcrM,9572
-jmcomic-2.6.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-jmcomic-2.6.6.dist-info/entry_points.txt,sha256=tRbQltaGSBjejI0c9jYt-4SXQMd5nSDHcMvHmuTy4ow,44
-jmcomic-2.6.6.dist-info/top_level.txt,sha256=puvVMFYJqIbd6NOTMEvOyugMTT8woBfSQyxEBan3zY4,8
-jmcomic-2.6.6.dist-info/RECORD,,