cobweb-launcher 1.3.10__tar.gz → 1.3.12__tar.gz

{cobweb-launcher-1.3.10/cobweb_launcher.egg-info → cobweb-launcher-1.3.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.10
+Version: 1.3.12
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.10 → cobweb-launcher-1.3.12}/cobweb/base/__init__.py

@@ -59,8 +59,9 @@ class TaskQueue:
                 else:
                     raise TypeError(f"{crawler_func.__name__} function return type isn't supported")
                 TaskQueue.DOT.build(
-                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}:{tk.__class__.__name__}",
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
                     cost_time=round(time.time() - start_time, 2),
+                    process_task_type=tk.__class__.__name__,
                     **tk.to_dict
                 )
         except Exception as e:

{cobweb-launcher-1.3.10 → cobweb-launcher-1.3.12}/cobweb/crawlers/crawler.py

@@ -1,4 +1,5 @@
 import json
+import os
 import time
 import threading
 from typing import Union, Callable, Mapping
@@ -52,14 +53,21 @@ class Crawler(threading.Thread):
         thread_sleep = 0.1
         if TaskQueue.REQUEST.length >= self.request_queue_size:
             thread_sleep = 5
-        elif seed_info := TaskQueue.TODO.pop():
-            member, priority = seed_info
-            seed = Seed(member, priority=priority)
+        elif seed := TaskQueue.TODO.pop():
+            # member, priority = seed_info
+            # seed = Seed(member, priority=priority)
             if seed.params.retry > self.spider_max_retries:
-                seed.params.seed_status = DealModel.fail
+                # seed.params.seed_status = DealModel.fail
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    # cost_time=round(time.time() - start_time, 2),
+                    process_task_type=seed.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    **seed.to_dict
+                )
             else:
                 TaskQueue.process_task(seed, self.request)
-            TaskQueue.DELETE.push(member)
+            TaskQueue.DELETE.push(seed.seed)
         time.sleep(thread_sleep)
 
     @Decorators.pause
@@ -68,21 +76,40 @@ class Crawler(threading.Thread):
         if TaskQueue.RESPONSE.length >= self.download_queue_size:
             thread_sleep = 5
             # logger.info(f"download queue is full, sleep {thread_sleep}s")
-        elif request_info := TaskQueue.DOWNLOAD.pop():
-            member, priority = request_info
-            request_setting = json.loads(member)
-            request_item = Request(seed=member, **request_setting)
-            TaskQueue.process_task(request_item, self.download)
+        elif request_item := TaskQueue.DOWNLOAD.pop():
+            # member, priority = request_info
+            #
+            # request_setting = json.loads(member)
+            # request_item = Request(seed=member, **request_setting)
+            if request_item.params.retry > self.spider_max_retries:
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    # cost_time=round(time.time() - start_time, 2),
+                    process_task_type=request_item.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    **request_item.to_dict
+                )
+                TaskQueue.DONE.push(request_item.seed)
+            else:
+                TaskQueue.process_task(request_item, self.download)
         time.sleep(thread_sleep)
 
     @Decorators.pause
     def build_parse_item(self):
         thread_sleep = 0.1
         if TaskQueue.UPLOAD.length >= self.upload_queue_size:
-            # logger.info(f"upload queue is full, sleep {thread_sleep}s")
             thread_sleep = 5
-        if response_item := TaskQueue.RESPONSE.pop():
-            TaskQueue.process_task(response_item, self.parse)
+        elif response_item := TaskQueue.RESPONSE.pop():
+            if response_item.params.retry > self.spider_max_retries:
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    process_task_type=response_item.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    **response_item.to_dict
+                )
+                TaskQueue.DONE.push(response_item.seed)
+            else:
+                TaskQueue.process_task(response_item, self.parse)
         time.sleep(thread_sleep)
 
 

{cobweb-launcher-1.3.10 → cobweb-launcher-1.3.12}/cobweb/launchers/launcher.py

@@ -60,6 +60,7 @@ class Launcher(threading.Thread):
         self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
         self.delete_queue_size = setting.DELETE_QUEUE_SIZE
         self.done_queue_size = setting.DONE_QUEUE_SIZE
+        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
 
         self.spider_thread_num = setting.SPIDER_THREAD_NUM
 

{cobweb-launcher-1.3.10 → cobweb-launcher-1.3.12}/cobweb/launchers/launcher_api.py

@@ -1,6 +1,6 @@
 import time
 
-from cobweb.base import TaskQueue, Decorators
+from cobweb.base import TaskQueue, Decorators, Seed, Request
 from cobweb.schedulers import ApiScheduler
 from .launcher import Launcher
 
@@ -16,13 +16,13 @@ class LauncherApi(Launcher):
     @Decorators.stop
     def _schedule(self):
         thread_sleep = self.scheduling_wait_time
-        for q, key, size, item_info in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"]),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"]),
+        for q, key, size, item_info, Cls in [
+            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"], Seed),
+            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"], Request),
         ]:
             if q.length < size:
                 for member, priority in self._scheduler.schedule(key, self.scheduling_size):
-                    q.push((member, priority), direct_insertion=True)
+                    q.push(Cls(member, priority=priority))
                     self.add_working_item(key.split(":")[-1], member, priority)
                 thread_sleep = 0.1
         time.sleep(thread_sleep)

{cobweb-launcher-1.3.10 → cobweb-launcher-1.3.12/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.10
+Version: 1.3.12
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.10 → cobweb-launcher-1.3.12}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.3.10",
+    version="1.3.12",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",