cobweb-launcher 1.3.11__py3-none-any.whl → 1.3.12__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -1,4 +1,5 @@
 import json
+import os
 import time
 import threading
 from typing import Union, Callable, Mapping
@@ -52,14 +53,21 @@ class Crawler(threading.Thread):
         thread_sleep = 0.1
         if TaskQueue.REQUEST.length >= self.request_queue_size:
             thread_sleep = 5
-        elif seed_info := TaskQueue.TODO.pop():
-            member, priority = seed_info
-            seed = Seed(member, priority=priority)
+        elif seed := TaskQueue.TODO.pop():
+            # member, priority = seed_info
+            # seed = Seed(member, priority=priority)
             if seed.params.retry > self.spider_max_retries:
-                seed.params.seed_status = DealModel.fail
+                # seed.params.seed_status = DealModel.fail
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    # cost_time=round(time.time() - start_time, 2),
+                    process_task_type=seed.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    **seed.to_dict
+                )
             else:
                 TaskQueue.process_task(seed, self.request)
-            TaskQueue.DELETE.push(member)
+            TaskQueue.DELETE.push(seed.seed)
         time.sleep(thread_sleep)
 
     @Decorators.pause
@@ -68,21 +76,40 @@ class Crawler(threading.Thread):
         if TaskQueue.RESPONSE.length >= self.download_queue_size:
             thread_sleep = 5
             # logger.info(f"download queue is full, sleep {thread_sleep}s")
-        elif request_info := TaskQueue.DOWNLOAD.pop():
-            member, priority = request_info
-            request_setting = json.loads(member)
-            request_item = Request(seed=member, **request_setting)
-            TaskQueue.process_task(request_item, self.download)
+        elif request_item := TaskQueue.DOWNLOAD.pop():
+            # member, priority = request_info
+            #
+            # request_setting = json.loads(member)
+            # request_item = Request(seed=member, **request_setting)
+            if request_item.params.retry > self.spider_max_retries:
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    # cost_time=round(time.time() - start_time, 2),
+                    process_task_type=request_item.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    **request_item.to_dict
+                )
+                TaskQueue.DONE.push(request_item.seed)
+            else:
+                TaskQueue.process_task(request_item, self.download)
         time.sleep(thread_sleep)
 
     @Decorators.pause
     def build_parse_item(self):
         thread_sleep = 0.1
         if TaskQueue.UPLOAD.length >= self.upload_queue_size:
-            # logger.info(f"upload queue is full, sleep {thread_sleep}s")
             thread_sleep = 5
-        if response_item := TaskQueue.RESPONSE.pop():
-            TaskQueue.process_task(response_item, self.parse)
+        elif response_item := TaskQueue.RESPONSE.pop():
+            if response_item.params.retry > self.spider_max_retries:
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    process_task_type=response_item.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    **response_item.to_dict
+                )
+                TaskQueue.DONE.push(response_item.seed)
+            else:
+                TaskQueue.process_task(response_item, self.parse)
         time.sleep(thread_sleep)
 
 

cobweb/launchers/launcher.py

@@ -60,6 +60,7 @@ class Launcher(threading.Thread):
         self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
         self.delete_queue_size = setting.DELETE_QUEUE_SIZE
         self.done_queue_size = setting.DONE_QUEUE_SIZE
+        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
 
         self.spider_thread_num = setting.SPIDER_THREAD_NUM
 

cobweb/launchers/launcher_api.py

@@ -1,6 +1,6 @@
 import time
 
-from cobweb.base import TaskQueue, Decorators
+from cobweb.base import TaskQueue, Decorators, Seed, Request
 from cobweb.schedulers import ApiScheduler
 from .launcher import Launcher
 
@@ -16,13 +16,13 @@ class LauncherApi(Launcher):
     @Decorators.stop
     def _schedule(self):
         thread_sleep = self.scheduling_wait_time
-        for q, key, size, item_info in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"]),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"]),
+        for q, key, size, item_info, Cls in [
+            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"], Seed),
+            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"], Request),
         ]:
             if q.length < size:
                 for member, priority in self._scheduler.schedule(key, self.scheduling_size):
-                    q.push((member, priority), direct_insertion=True)
+                    q.push(Cls(member, priority=priority))
                     self.add_working_item(key.split(":")[-1], member, priority)
                 thread_sleep = 0.1
         time.sleep(thread_sleep)

{cobweb_launcher-1.3.11.dist-info → cobweb_launcher-1.3.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.11
+Version: 1.3.12
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.3.11.dist-info → cobweb_launcher-1.3.12.dist-info}/RECORD

@@ -11,16 +11,16 @@ cobweb/base/request.py,sha256=acGm3OzxsPed5VUTk7D9eeHZPMh7KUNQRUv44G5znZg,2659
 cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=PN5J4gKPEXylwyQeSGOBfauxHktxFr7RJe8nVX1hBw4,2987
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
-cobweb/crawlers/crawler.py,sha256=ZQ6yVA1EaQRdKJEY3DNqShzp9HPMwlSXapnsRW9E5Wc,2987
+cobweb/crawlers/crawler.py,sha256=G81gh_Rm0kypjnMKq0S5lrV39zu0zijtinJrQhwdWgI,4233
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
 cobweb/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=m_XNG2bWuMbirPt3d0_s-Ezl1xycfUxeqZnwq_kkfuo,116
-cobweb/launchers/launcher.py,sha256=O7HjZ2lewsZ5IIiZ2vhCAqGidJ1JKMxS5fs4KlslBkA,7332
+cobweb/launchers/launcher.py,sha256=dBHbm8Hj87CMCx5F9ZVo1oDPeW3L89d5Lu3PP9utYho,7393
 cobweb/launchers/launcher_air.py,sha256=yPr395HVIIHAq6lqRcYJu7c0KkfO9V8O-2sn0hC96p0,2990
-cobweb/launchers/launcher_api.py,sha256=TfLrLXazFWsOJLI7caMGfZozCttL1WTwTo3uUpN_FV0,3370
+cobweb/launchers/launcher_api.py,sha256=vpwFxB1azgBk1bS7VhX3jOprQS8fl6Iu_5M-Y3QT67A,3394
 cobweb/launchers/launcher_pro.py,sha256=2H-TcvQx-ga78GLNTa-GXMLYAj9nEeCJSWf8xl-1ISQ,3374
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=Pycm22bHId9a3gdP81D5y7SsuMndYooTb5n4zQxP7dM,1321
@@ -33,8 +33,8 @@ cobweb/utils/__init__.py,sha256=YvD4mIDBd9jmGA6WJBcwkgDU2jRFNBCEbarZCSUBAHE,114
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/oss.py,sha256=6Qlhdde7CcwD69bBe2rGWHY3-aptG9NXB_DZLhjgDRQ,3553
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.3.11.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.3.11.dist-info/METADATA,sha256=5GKTN8jXVIcQY-kAcbc2-o8NPPImj_9KFrwaBXKhXMY,6510
-cobweb_launcher-1.3.11.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.3.11.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.3.11.dist-info/RECORD,,
+cobweb_launcher-1.3.12.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.3.12.dist-info/METADATA,sha256=NJlheF7Uwt4i6XKbgI6v-mSuyIk5PNSmU-H7fy3VCC8,6510
+cobweb_launcher-1.3.12.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.3.12.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.3.12.dist-info/RECORD,,