cobweb-launcher 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

cobweb/crawlers/base_crawler.py

@@ -58,18 +58,18 @@ class Crawler(threading.Thread):
                 self.launcher_queue['done'].push(seed)
                 continue
 
-            item = self.request(seed)
+            seed_detail_log_info = download_log_info(seed.to_dict)
 
-            if isinstance(item, Request):
+            try:
+                item = self.request(seed)
 
-                download_iterators = self.download(item)
+                if isinstance(item, Request):
 
-                if not isgenerator(download_iterators):
-                    raise TypeError("download function isn't a generator")
+                    download_iterators = self.download(item)
 
-                seed_detail_log_info = download_log_info(seed.to_dict)
+                    if not isgenerator(download_iterators):
+                        raise TypeError("download function isn't a generator")
 
-                try:
                     for it in download_iterators:
                         if isinstance(it, Response):
                             response_detail_log_info = download_log_info(it.to_dict)
@@ -104,17 +104,22 @@ class Crawler(threading.Thread):
                         else:
                             raise TypeError("yield value type error!")
 
-                except Exception as e:
-                    logger.info(LogTemplate.download_exception.format(
-                        detail=seed_detail_log_info, retry=seed.params.retry,
-                        priority=seed.params.priority, seed_version=seed.params.seed_version,
-                        identifier=seed.params.identifier, exception=e
-                    ))
-                    seed.params.retry += 1
-                    self.launcher_queue['todo'].push(seed)
-
-            elif isinstance(item, BaseItem):
-                self.upload_queue.push(item)
+                elif isinstance(item, BaseItem):
+                    self.upload_queue.push(item)
+                else:
+                    raise TypeError(
+                        f"request func return value type error!"
+                        f"item.__class__ is  {item.__class__.__name__}"
+                    )
+            except Exception as e:
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info, retry=seed.params.retry,
+                    priority=seed.params.priority, seed_version=seed.params.seed_version,
+                    identifier=seed.params.identifier, exception=e
+                ))
+                seed.params.retry += 1
+                self.launcher_queue['todo'].push(seed)
+
 
     def run(self):
         for index in range(self.spider_thread_num):

cobweb/db/redis_db.py

@@ -4,8 +4,9 @@ from cobweb import setting
 
 class RedisDB:
 
-    def __init__(self):
-        pool = redis.ConnectionPool(**setting.REDIS_CONFIG)
+    def __init__(self, **kwargs):
+        redis_config = kwargs or setting.REDIS_CONFIG
+        pool = redis.ConnectionPool(**redis_config)
         self._client = redis.Redis(connection_pool=pool)
 
     def setnx(self, name, value=""):

cobweb/launchers/launcher_pro.py

@@ -140,22 +140,23 @@ class LauncherPro(Launcher):
         while not self._stop.is_set():
             queue_not_empty_count = 0
             pooling_wait_seconds = 30
-            if not self._db.zcard(self._todo):
-                for q in self.__LAUNCHER_QUEUE__.values():
-                    if q.length != 0:
-                        queue_not_empty_count += 1
-                if self._pause.is_set():
-                    self._pause.clear()
-                    self._execute()
-                elif queue_not_empty_count == 0:
-                    pooling_wait_seconds = 3
-                    check_emtpy_times += 1
-                else:
-                    check_emtpy_times = 0
-                if check_emtpy_times > 2:
-                    check_emtpy_times = 0
-                    self.__DOING__ = {}
-                    self._pause.set()
+            for q in self.__LAUNCHER_QUEUE__.values():
+                if q.length != 0:
+                    queue_not_empty_count += 1
+            if self._pause.is_set():
+                self._pause.clear()
+                self._execute()
+            elif queue_not_empty_count == 0:
+                pooling_wait_seconds = 3
+                check_emtpy_times += 1
+            else:
+                check_emtpy_times = 0
+
+            if not self._db.zcard(self._todo) and check_emtpy_times > 2:
+                check_emtpy_times = 0
+                self.__DOING__ = {}
+                self._pause.set()
+
             if not self._pause.is_set():
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,

{cobweb_launcher-1.1.4.dist-info → cobweb_launcher-1.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.4
+Version: 1.1.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.4.dist-info → cobweb_launcher-1.1.6.dist-info}/RECORD

@@ -10,23 +10,23 @@ cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
 cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
 cobweb/crawlers/__init__.py,sha256=_HAXBg7Sq8fsDGSjDm3AQz9aQtLZONpt5b8dSe607mI,91
-cobweb/crawlers/base_crawler.py,sha256=YtFju88e3NIzGgYRSEIiMD-H4fPa_IDlgFxtxMiADzk,4963
+cobweb/crawlers/base_crawler.py,sha256=uR1wQ2sJpFovNoAK52293rF03O-jNbv24P5QoNt1tW0,5169
 cobweb/crawlers/file_crawler.py,sha256=LTiHaxhEiJyiAGgodO3an8AYf_y88AeMoFcKae3Vx_M,8381
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
-cobweb/db/redis_db.py,sha256=uML8WWY5GVT-mVeCBL40Ip-0Bso6UHkAdn2cQueQxCg,4248
+cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
 cobweb/launchers/launcher.py,sha256=O6Kkvqk-0kOxJom8YO9zW18e_2eMYrA5RTS9Xy4TW5k,5665
-cobweb/launchers/launcher_pro.py,sha256=3B7JjT2wk-q_Q9yCIw36OIS0eGVNBFRV5F6WLzH4FKo,6620
+cobweb/launchers/launcher_pro.py,sha256=2N_GU3PVv1CP-ZBPOhh2xAm_KkPHx4SVC0JPW1t3JB4,6550
 cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=uD5aN2oVYImit3amE6TjxWMaTAcbAh9dCnpIQhf4M9Q,3238
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.4.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.4.dist-info/METADATA,sha256=Ta-du5glAOJXADYADS6Ew8003nssyTzjcAglz4o6WBM,1245
-cobweb_launcher-1.1.4.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.4.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.4.dist-info/RECORD,,
+cobweb_launcher-1.1.6.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.6.dist-info/METADATA,sha256=U7bo6LBCeF5nZMKm7zUNyUHt40cNvlkhr2lgK7LSkQQ,1245
+cobweb_launcher-1.1.6.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.6.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.6.dist-info/RECORD,,