cobweb-launcher 1.1.19__py3-none-any.whl → 1.1.21__py3-none-any.whl

cobweb/crawlers/base_crawler.py

@@ -1,4 +1,5 @@
 import threading
+import time
 
 from inspect import isgenerator
 from typing import Union, Callable, Mapping
@@ -81,15 +82,22 @@ class Crawler(threading.Thread):
                 if not isgenerator(request_iterators):
                     raise TypeError("request function isn't a generator!")
 
+                iterator_status = False
+
                 for request_item in request_iterators:
 
+                    iterator_status = True
+
                     if isinstance(request_item, Request):
+                        iterator_status = False
                         download_iterators = self.download(request_item)
                         if not isgenerator(download_iterators):
                             raise TypeError("download function isn't a generator")
 
                         for download_item in download_iterators:
+                            iterator_status = True
                             if isinstance(download_item, Response):
+                                iterator_status = False
                                 logger.info(LogTemplate.download_info.format(
                                     detail=seed_detail_log_info,
                                     retry=seed.params.retry,
@@ -103,6 +111,7 @@ class Crawler(threading.Thread):
                                 if not isgenerator(parse_iterators):
                                     raise TypeError("parse function isn't a generator")
                                 for parse_item in parse_iterators:
+                                    iterator_status = True
                                     if isinstance(parse_item, Response):
                                         raise TypeError("upload_item can't be a Response instance")
                                     self.distribute(parse_item, seed)
@@ -110,6 +119,10 @@ class Crawler(threading.Thread):
                                 self.distribute(download_item, seed)
                     else:
                         self.distribute(request_item, seed)
+
+                if not iterator_status:
+                    raise ValueError("request/download/parse function yield value error!")
+
             except Exception as e:
                 logger.info(LogTemplate.download_exception.format(
                     detail=seed_detail_log_info,
@@ -120,6 +133,8 @@ class Crawler(threading.Thread):
                 ))
                 seed.params.retry += 1
                 self.launcher_queue['todo'].push(seed)
+            finally:
+                time.sleep(0.1)
 
     def run(self):
         for index in range(self.spider_thread_num):

cobweb/launchers/launcher_pro.py

@@ -125,12 +125,13 @@ class LauncherPro(Launcher):
                     seeds.append(seed.to_string)
             if seeds:
                 self._db.zrem(self._todo, *seeds)
+                self._remove_doing_seeds(seeds)
             if s_seeds:
                 self._db.done([self._todo, self._done], *s_seeds)
+                self._remove_doing_seeds(s_seeds)
             if f_seeds:
                 self._db.done([self._todo, self._fail], *f_seeds)
-
-            self._remove_doing_seeds(seeds)
+                self._remove_doing_seeds(f_seeds)
 
             if status:
                 time.sleep(self._done_queue_wait_seconds)

{cobweb_launcher-1.1.19.dist-info → cobweb_launcher-1.1.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.19
+Version: 1.1.21
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.19.dist-info → cobweb_launcher-1.1.21.dist-info}/RECORD

@@ -10,7 +10,7 @@ cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
 cobweb/crawlers/__init__.py,sha256=Rr3DTjD-abMA1_FYcQJZYNvQvcWMuEVcsIU6duqHrw4,75
-cobweb/crawlers/base_crawler.py,sha256=6_v71I-opx6JhSKFWa7qF__ivzKwRwAgwGI8nwx3YMQ,5154
+cobweb/crawlers/base_crawler.py,sha256=05uIIPs_1RdRKPfEcS8zAwgXbBODOJqAJUjWsMw7jQo,5651
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
@@ -18,15 +18,15 @@ cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
 cobweb/launchers/launcher.py,sha256=LQdlpaF4fafEX01_3B9CB6hD-6YMPag6QOGl9rDprNE,5707
-cobweb/launchers/launcher_pro.py,sha256=lBHj5OaLNRbHwohI3tRRL3WZQ0uOolZAb8HdoTftq40,6594
+cobweb/launchers/launcher_pro.py,sha256=VH_HnSQGAzxod6aFB3ZySi6-5Yp4vI-VE3u1XC1dfWg,6697
 cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=qAl05ybL2Jp6KFjHDHVMfmeBHQmDKPtZleHjHAY7LZc,3277
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.19.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.19.dist-info/METADATA,sha256=e-o4DJ4zQ8qbHuR3lllCJOGcAdxACG_je1w_o5hcyTs,1246
-cobweb_launcher-1.1.19.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.19.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.19.dist-info/RECORD,,
+cobweb_launcher-1.1.21.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.21.dist-info/METADATA,sha256=jPQSJUuw_07x61hjH52AhxralQQjb8yDVnR0ajtccmU,1246
+cobweb_launcher-1.1.21.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.21.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.21.dist-info/RECORD,,