cobweb-launcher 1.2.42__tar.gz → 1.2.43__tar.gz

{cobweb-launcher-1.2.42/cobweb_launcher.egg-info → cobweb-launcher-1.2.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.42
+Version: 1.2.43
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.42 → cobweb-launcher-1.2.43}/cobweb/crawlers/crawler.py

@@ -8,6 +8,7 @@ from urllib.parse import urlparse
 
 import urllib3
 from requests import HTTPError, Response as Res
+from requests.exceptions import ChunkedEncodingError
 
 from cobweb.constant import DealModel, LogTemplate
 from cobweb.base import (
@@ -165,7 +166,7 @@ class Crawler(threading.Thread):
 
                 if not iterator_status:
                     raise ValueError("request/download/parse function yield value error!")
-            except (HTTPError, urllib3.exceptions.HTTPError, urllib3.exceptions.PoolError) as e:
+            except Exception as e:
                 exception_msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
                 url = seed.url
                 status = str(e)
@@ -189,19 +190,19 @@ class Crawler(threading.Thread):
                 seed.params.retry += 1
                 self._set_seed(seed)
                 # time.sleep(self.time_sleep * seed.params.retry)
-            except Exception as e:
-                logger.info(LogTemplate.download_exception.format(
-                    detail=seed_detail_log_info,
-                    retry=seed.params.retry,
-                    priority=seed.params.priority,
-                    seed_version=seed.params.seed_version,
-                    identifier=seed.identifier or "",
-                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                ))
-                seed.params.retry += 1
-                # self._todo.push(seed)
-                self._set_seed(seed)
-                # time.sleep(self.time_sleep * seed.params.retry)
+            # except Exception as e:
+            #     logger.info(LogTemplate.download_exception.format(
+            #         detail=seed_detail_log_info,
+            #         retry=seed.params.retry,
+            #         priority=seed.params.priority,
+            #         seed_version=seed.params.seed_version,
+            #         identifier=seed.identifier or "",
+            #         exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
+            #     ))
+            #     seed.params.retry += 1
+            #     # self._todo.push(seed)
+            #     self._set_seed(seed)
+            #     # time.sleep(self.time_sleep * seed.params.retry)
             finally:
                 time.sleep(0.1)
         logger.info("spider thread close")

{cobweb-launcher-1.2.42 → cobweb-launcher-1.2.43/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.42
+Version: 1.2.43
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.42 → cobweb-launcher-1.2.43}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.42",
+    version="1.2.43",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",