cobweb-launcher 1.0.9__py3-none-any.whl → 1.1.1__py3-none-any.whl

cobweb/crawlers/base_crawler.py

@@ -37,7 +37,7 @@ class Crawler(threading.Thread):
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
         response = item.download()
-        yield Response(item.seed, response)
+        yield Response(item.seed, response, **item.to_dict)
 
     @staticmethod
     def parse(item: Response) -> BaseItem:
@@ -54,6 +54,7 @@ class Crawler(threading.Thread):
                 continue
 
             elif seed.params.retry >= self.max_retries:
+                seed.params.identifier = DealModel.fail
                 self.launcher_queue['done'].push(seed)
                 continue
 

cobweb/db/redis_db.py

@@ -126,3 +126,4 @@ class RedisDB:
         self.execute_lua(lua_script, keys, *args)
 
 
+

cobweb/launchers/launcher_pro.py

@@ -16,7 +16,6 @@ class LauncherPro(Launcher):
         self._fail = "{%s:%s}:fail" % (project, task)
         self._heartbeat = "heartbeat:%s_%s" % (project, task)
         self._reset_lock = "lock:reset:%s_%s" % (project, task)
-        self._heartbeat_lock = "lock:heartbeat:%s_%s" % (project, task)
         self._db = RedisDB()
 
         self._heartbeat_start_event = threading.Event()
@@ -29,10 +28,8 @@ class LauncherPro(Launcher):
     def _execute_heartbeat(self):
         while not self._stop.is_set():
             if self._heartbeat_start_event.is_set():
-                if self._db.lock(self._heartbeat_lock, t=1):
-                    self._db.setex(self._heartbeat, 3)
-                time.sleep(1)
-            time.sleep(0.5)
+                self._db.setex(self._heartbeat, 3)
+            time.sleep(2)
 
     def _reset(self):
         """

cobweb/setting.py

@@ -23,8 +23,8 @@ OSS_BUCKET = os.getenv("OSS_BUCKET")
 OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
 OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
 OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
+OSS_CHUNK_SIZE = 10 * 1024 ** 2
 OSS_MIN_UPLOAD_SIZE = 1024
-OSS_CHUNK_SIZE = 1024 ** 2
 
 # 采集器选择
 CRAWLER = "cobweb.crawlers.CrawlerAir"

{cobweb_launcher-1.0.9.dist-info → cobweb_launcher-1.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.0.9
+Version: 1.1.1
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.0.9.dist-info → cobweb_launcher-1.1.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=IkGcdTU6fEBNyzWowcJSSMdErntFM1kmu4WUp1BgImU,45
 cobweb/constant.py,sha256=Aw2ES_nohVRLTWylZp6WMiUAlgyw4kLbae7LpwdZ5y4,1867
-cobweb/setting.py,sha256=fYK8KgrZYwSoJC9ywsopAtV0mdRw0_MGAoYORgET01M,1949
+cobweb/setting.py,sha256=9kiyVbFC60Wvbh2tfo52XXELYXhhJZeG5hPLhZYbHbQ,1954
 cobweb/base/__init__.py,sha256=diiK5MygQaWjlWNLbW6eUIg-93O6glMGC9WLNM5jyOc,209
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -10,23 +10,23 @@ cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
 cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
 cobweb/crawlers/__init__.py,sha256=1sMhQ0-NJxiff3IqF2aMCXkSXcJFzzoCKIayQ5go4aI,71
-cobweb/crawlers/base_crawler.py,sha256=snfjGYV9oZl3UnrxUcpAD9cK0n_rPeuyUYQkU1aCBrA,4891
+cobweb/crawlers/base_crawler.py,sha256=YtFju88e3NIzGgYRSEIiMD-H4fPa_IDlgFxtxMiADzk,4963
 cobweb/crawlers/file_crawler.py,sha256=AeK22bakMbwDfqrITlX6ssvR2KSpHQ3s8YEuzX_Bluk,8670
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
-cobweb/db/redis_db.py,sha256=XE1ebIi_4e0KBKiyPdKX7l2lSgV5vhMwEhNDlAcsccU,4247
+cobweb/db/redis_db.py,sha256=uML8WWY5GVT-mVeCBL40Ip-0Bso6UHkAdn2cQueQxCg,4248
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
 cobweb/launchers/launcher.py,sha256=O6Kkvqk-0kOxJom8YO9zW18e_2eMYrA5RTS9Xy4TW5k,5665
-cobweb/launchers/launcher_pro.py,sha256=GlDpyP1XAY2bX5SuSBn3920D5OKNigQgLnJfu6QOmPw,6760
+cobweb/launchers/launcher_pro.py,sha256=nGPnkQ6TedYNf3Liugqt8yKG48T3lNmgFd1BVptSdHw,6591
 cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=uD5aN2oVYImit3amE6TjxWMaTAcbAh9dCnpIQhf4M9Q,3238
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.0.9.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.0.9.dist-info/METADATA,sha256=OGtzwPd4tfBxFIU-KhjR4JPYG3sF_3Em1Av3Ah0tOCs,1245
-cobweb_launcher-1.0.9.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.0.9.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.0.9.dist-info/RECORD,,
+cobweb_launcher-1.1.1.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.1.dist-info/METADATA,sha256=bTZHpxiQt-1ZSP3div4SCSHDSUmJpLej_L7znSM1mkI,1245
+cobweb_launcher-1.1.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.1.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.1.dist-info/RECORD,,