cobweb-launcher 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

cobweb/__init__.py

@@ -1 +1,2 @@
 from .launchers import Launcher, LauncherPro
+from .constant import Crawler

cobweb/constant.py

@@ -1,4 +1,10 @@
 
+class Crawler:
+
+    default = "cobweb.crawlers.Crawler"
+    file_air = "cobweb.crawlers.FileCrawlerAir"
+    file_pro = "cobweb.crawlers.FileCrawlerPro"
+
 
 class LauncherModel:
     task = "launcher model: task"

cobweb/crawlers/__init__.py

@@ -1,2 +1,2 @@
 from .base_crawler import Crawler
-from .file_crawler import FileCrawlerAir
+from .file_crawler import FileCrawlerAir, FileCrawlerPro

cobweb/crawlers/base_crawler.py

@@ -58,18 +58,18 @@ class Crawler(threading.Thread):
                 self.launcher_queue['done'].push(seed)
                 continue
 
-            item = self.request(seed)
+            seed_detail_log_info = download_log_info(seed.to_dict)
 
-            if isinstance(item, Request):
+            try:
+                item = self.request(seed)
 
-                download_iterators = self.download(item)
+                if isinstance(item, Request):
 
-                if not isgenerator(download_iterators):
-                    raise TypeError("download function isn't a generator")
+                    download_iterators = self.download(item)
 
-                seed_detail_log_info = download_log_info(seed.to_dict)
+                    if not isgenerator(download_iterators):
+                        raise TypeError("download function isn't a generator")
 
-                try:
                     for it in download_iterators:
                         if isinstance(it, Response):
                             response_detail_log_info = download_log_info(it.to_dict)
@@ -104,17 +104,22 @@ class Crawler(threading.Thread):
                         else:
                             raise TypeError("yield value type error!")
 
-                except Exception as e:
-                    logger.info(LogTemplate.download_exception.format(
-                        detail=seed_detail_log_info, retry=seed.params.retry,
-                        priority=seed.params.priority, seed_version=seed.params.seed_version,
-                        identifier=seed.params.identifier, exception=e
-                    ))
-                    seed.params.retry += 1
-                    self.launcher_queue['todo'].push(seed)
-
-            elif isinstance(item, BaseItem):
-                self.upload_queue.push(item)
+                elif isinstance(item, BaseItem):
+                    self.upload_queue.push(item)
+                else:
+                    raise TypeError(
+                        f"request func return value type error!"
+                        f"item.__class__ is  {item.__class__.__name__}"
+                    )
+            except Exception as e:
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info, retry=seed.params.retry,
+                    priority=seed.params.priority, seed_version=seed.params.seed_version,
+                    identifier=seed.params.identifier, exception=e
+                ))
+                seed.params.retry += 1
+                self.launcher_queue['todo'].push(seed)
+
 
     def run(self):
         for index in range(self.spider_thread_num):

cobweb/db/redis_db.py

@@ -4,8 +4,9 @@ from cobweb import setting
 
 class RedisDB:
 
-    def __init__(self):
-        pool = redis.ConnectionPool(**setting.REDIS_CONFIG)
+    def __init__(self, **kwargs):
+        redis_config = kwargs or setting.REDIS_CONFIG
+        pool = redis.ConnectionPool(**redis_config)
         self._client = redis.Redis(connection_pool=pool)
 
     def setnx(self, name, value=""):

{cobweb_launcher-1.1.3.dist-info → cobweb_launcher-1.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.3
+Version: 1.1.5
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.3.dist-info → cobweb_launcher-1.1.5.dist-info}/RECORD

@@ -1,5 +1,5 @@
-cobweb/__init__.py,sha256=IkGcdTU6fEBNyzWowcJSSMdErntFM1kmu4WUp1BgImU,45
-cobweb/constant.py,sha256=Aw2ES_nohVRLTWylZp6WMiUAlgyw4kLbae7LpwdZ5y4,1867
+cobweb/__init__.py,sha256=UjV-xEAOr7IDUEWp8mkGvbQlOxF8JfG677Ru_OkAZZc,75
+cobweb/constant.py,sha256=-h7w2xN7WV9vrikN54TW4oSHUk0HruWpg9g2wjUoeDk,2020
 cobweb/setting.py,sha256=MRjqm75EAhqFi_6We5piF_cTCfRJojRXDKkLhP3yhvo,1951
 cobweb/base/__init__.py,sha256=diiK5MygQaWjlWNLbW6eUIg-93O6glMGC9WLNM5jyOc,209
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
@@ -9,11 +9,11 @@ cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
 cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
 cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
-cobweb/crawlers/__init__.py,sha256=Rr3DTjD-abMA1_FYcQJZYNvQvcWMuEVcsIU6duqHrw4,75
-cobweb/crawlers/base_crawler.py,sha256=YtFju88e3NIzGgYRSEIiMD-H4fPa_IDlgFxtxMiADzk,4963
+cobweb/crawlers/__init__.py,sha256=_HAXBg7Sq8fsDGSjDm3AQz9aQtLZONpt5b8dSe607mI,91
+cobweb/crawlers/base_crawler.py,sha256=uR1wQ2sJpFovNoAK52293rF03O-jNbv24P5QoNt1tW0,5169
 cobweb/crawlers/file_crawler.py,sha256=LTiHaxhEiJyiAGgodO3an8AYf_y88AeMoFcKae3Vx_M,8381
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
-cobweb/db/redis_db.py,sha256=uML8WWY5GVT-mVeCBL40Ip-0Bso6UHkAdn2cQueQxCg,4248
+cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
@@ -25,8 +25,8 @@ cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzX
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=uD5aN2oVYImit3amE6TjxWMaTAcbAh9dCnpIQhf4M9Q,3238
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.3.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.3.dist-info/METADATA,sha256=y15_B4PsJqjnbx-HjdzFWBxqT6Z3R5LQSlPUI4DnoKE,1245
-cobweb_launcher-1.1.3.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.3.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.3.dist-info/RECORD,,
+cobweb_launcher-1.1.5.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.5.dist-info/METADATA,sha256=F5vvm0pngBeivTOVQ_srp-7epa3gHadaovtottKAIko,1245
+cobweb_launcher-1.1.5.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.5.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.5.dist-info/RECORD,,