cobweb-launcher 1.2.41__py3-none-any.whl → 1.2.43__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -6,7 +6,9 @@ from inspect import isgenerator
 from typing import Union, Callable, Mapping
 from urllib.parse import urlparse
 
+import urllib3
 from requests import HTTPError, Response as Res
+from requests.exceptions import ChunkedEncodingError
 
 from cobweb.constant import DealModel, LogTemplate
 from cobweb.base import (
@@ -164,29 +166,19 @@ class Crawler(threading.Thread):
 
                 if not iterator_status:
                     raise ValueError("request/download/parse function yield value error!")
-            except HTTPError as e:
-                if isinstance(e.response, Res):
+            except Exception as e:
+                exception_msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                url = seed.url
+                status = str(e)
+                if getattr(e, "response", None) and isinstance(e.response, Res):
                     url = e.response.request.url
                     status = e.response.status_code
-                    exception_msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                    self.loghub_dot.build(
-                        topic=urlparse(url).netloc,
-                        data_size=-1, cost_time=-1,
-                        status=status, url=url,
-                        msg=exception_msg
-                    )
-                logger.info(LogTemplate.download_exception.format(
-                    detail=seed_detail_log_info,
-                    retry=seed.params.retry,
-                    priority=seed.params.priority,
-                    seed_version=seed.params.seed_version,
-                    identifier=seed.identifier or "",
-                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                ))
-                seed.params.retry += 1
-                self._set_seed(seed)
-                time.sleep(self.time_sleep * seed.params.retry)
-            except Exception as e:
+                self.loghub_dot.build(
+                    topic=urlparse(url).netloc,
+                    data_size=-1, cost_time=-1,
+                    status=status, url=url,
+                    msg=exception_msg
+                )
                 logger.info(LogTemplate.download_exception.format(
                     detail=seed_detail_log_info,
                     retry=seed.params.retry,
@@ -196,9 +188,21 @@ class Crawler(threading.Thread):
                     exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
                 ))
                 seed.params.retry += 1
-                # self._todo.push(seed)
                 self._set_seed(seed)
                 # time.sleep(self.time_sleep * seed.params.retry)
+            # except Exception as e:
+            #     logger.info(LogTemplate.download_exception.format(
+            #         detail=seed_detail_log_info,
+            #         retry=seed.params.retry,
+            #         priority=seed.params.priority,
+            #         seed_version=seed.params.seed_version,
+            #         identifier=seed.identifier or "",
+            #         exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
+            #     ))
+            #     seed.params.retry += 1
+            #     # self._todo.push(seed)
+            #     self._set_seed(seed)
+            #     # time.sleep(self.time_sleep * seed.params.retry)
             finally:
                 time.sleep(0.1)
         logger.info("spider thread close")

{cobweb_launcher-1.2.41.dist-info → cobweb_launcher-1.2.43.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.41
+Version: 1.2.43
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.41.dist-info → cobweb_launcher-1.2.43.dist-info}/RECORD

@@ -13,7 +13,7 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=UojWdymPCwit0MOkqHsYRoe4hXyHdZhgh7-MBPfrhQo,8373
+cobweb/crawlers/crawler.py,sha256=pEukp5tC-axkzmcagPIpWPgmpxP0NHC1eu8iyJDFegA,8537
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -37,8 +37,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=PgsWdM-724Jy-MZWUsaygNWV-huqLMmdLgop7gaBxlo,872
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.41.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.41.dist-info/METADATA,sha256=ZuTN2RXJGQB6qWfjgTtcvwoVrjxvS6-ho0z7V9BTR8A,6510
-cobweb_launcher-1.2.41.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.41.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.41.dist-info/RECORD,,
+cobweb_launcher-1.2.43.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.43.dist-info/METADATA,sha256=GcKa3nUwsRKVxxoe2lKqHylsTYtXHxbveUMAizWtdJc,6510
+cobweb_launcher-1.2.43.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.43.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.43.dist-info/RECORD,,