cobweb-launcher 1.2.41__tar.gz → 1.2.42__tar.gz

{cobweb-launcher-1.2.41/cobweb_launcher.egg-info → cobweb-launcher-1.2.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.41
+Version: 1.2.42
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.41 → cobweb-launcher-1.2.42}/cobweb/crawlers/crawler.py

@@ -6,6 +6,7 @@ from inspect import isgenerator
 from typing import Union, Callable, Mapping
 from urllib.parse import urlparse
 
+import urllib3
 from requests import HTTPError, Response as Res
 
 from cobweb.constant import DealModel, LogTemplate
@@ -164,17 +165,19 @@ class Crawler(threading.Thread):
 
                 if not iterator_status:
                     raise ValueError("request/download/parse function yield value error!")
-            except HTTPError as e:
-                if isinstance(e.response, Res):
+            except (HTTPError, urllib3.exceptions.HTTPError, urllib3.exceptions.PoolError) as e:
+                exception_msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                url = seed.url
+                status = str(e)
+                if getattr(e, "response", None) and isinstance(e.response, Res):
                     url = e.response.request.url
                     status = e.response.status_code
-                    exception_msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                    self.loghub_dot.build(
-                        topic=urlparse(url).netloc,
-                        data_size=-1, cost_time=-1,
-                        status=status, url=url,
-                        msg=exception_msg
-                    )
+                self.loghub_dot.build(
+                    topic=urlparse(url).netloc,
+                    data_size=-1, cost_time=-1,
+                    status=status, url=url,
+                    msg=exception_msg
+                )
                 logger.info(LogTemplate.download_exception.format(
                     detail=seed_detail_log_info,
                     retry=seed.params.retry,
@@ -185,7 +188,7 @@ class Crawler(threading.Thread):
                 ))
                 seed.params.retry += 1
                 self._set_seed(seed)
-                time.sleep(self.time_sleep * seed.params.retry)
+                # time.sleep(self.time_sleep * seed.params.retry)
             except Exception as e:
                 logger.info(LogTemplate.download_exception.format(
                     detail=seed_detail_log_info,

{cobweb-launcher-1.2.41 → cobweb-launcher-1.2.42/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.41
+Version: 1.2.42
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.41 → cobweb-launcher-1.2.42}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.41",
+    version="1.2.42",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",