cobweb-launcher 1.3.11__tar.gz → 1.3.13__tar.gz

{cobweb-launcher-1.3.11/cobweb_launcher.egg-info → cobweb-launcher-1.3.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.11
+Version: 1.3.13
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.11 → cobweb-launcher-1.3.13}/cobweb/base/__init__.py

@@ -100,14 +100,15 @@ class Decorators:
     def pause(func):
         @wraps(func)
         def wrapper(self, *args, **kwargs):
-            while not self.pause.is_set():
-                try:
-                    func(self)
-                except Exception as e:
-                    logger.info(f"{func.__name__}: " + str(e))
-                finally:
-                    time.sleep(0.1)
-            logger.info(f"{func.__name__}: close!")
+            while not self.stop.is_set():
+                while not self.pause.is_set():
+                    try:
+                        func(self)
+                    except Exception as e:
+                        logger.info(f"{func.__name__}: " + str(e))
+                    finally:
+                        time.sleep(0.1)
+                # logger.info(f"{func.__name__}: close!")
 
         return wrapper
 

{cobweb-launcher-1.3.11 → cobweb-launcher-1.3.13}/cobweb/base/basic.py

@@ -142,7 +142,7 @@ class Request:
 
     def __init__(
             self,
-            url,
+            # url,
             seed,
             random_ua=True,
             check_status_code=True,
@@ -152,7 +152,7 @@ class Request:
             status=None,
             **kwargs
     ):
-        self.url = url
+        # self.url = url
         self.check_status_code = check_status_code
         self.request_setting = {}
 

{cobweb-launcher-1.3.11 → cobweb-launcher-1.3.13}/cobweb/crawlers/crawler.py

@@ -1,4 +1,5 @@
 import json
+import os
 import time
 import threading
 from typing import Union, Callable, Mapping
@@ -12,15 +13,15 @@ from cobweb.base import (
     ConsoleItem,
     Decorators,
     TaskQueue,
-    logger
 )
 from cobweb.constant import DealModel
 
 
 class Crawler(threading.Thread):
 
-    def __init__(self, pause, custom_func: Union[Mapping[str, Callable]]):
+    def __init__(self, stop, pause, custom_func: Union[Mapping[str, Callable]]):
         super().__init__()
+        self.stop = stop
         self.pause = pause
         for func_name, _callable in custom_func.items():
             if isinstance(_callable, Callable):
@@ -52,14 +53,20 @@ class Crawler(threading.Thread):
         thread_sleep = 0.1
         if TaskQueue.REQUEST.length >= self.request_queue_size:
             thread_sleep = 5
-        elif seed_info := TaskQueue.TODO.pop():
-            member, priority = seed_info
-            seed = Seed(member, priority=priority)
+        elif seed := TaskQueue.TODO.pop():
+            # member, priority = seed_info
+            # seed = Seed(member, priority=priority)
             if seed.params.retry > self.spider_max_retries:
-                seed.params.seed_status = DealModel.fail
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    process_task_type=seed.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    retries=seed.params.retry,
+                    **seed.to_dict
+                )
             else:
                 TaskQueue.process_task(seed, self.request)
-            TaskQueue.DELETE.push(member)
+            TaskQueue.DELETE.push(seed.seed)
         time.sleep(thread_sleep)
 
     @Decorators.pause
@@ -67,22 +74,37 @@ class Crawler(threading.Thread):
         thread_sleep = 0.1
         if TaskQueue.RESPONSE.length >= self.download_queue_size:
             thread_sleep = 5
-            # logger.info(f"download queue is full, sleep {thread_sleep}s")
-        elif request_info := TaskQueue.DOWNLOAD.pop():
-            member, priority = request_info
-            request_setting = json.loads(member)
-            request_item = Request(seed=member, **request_setting)
-            TaskQueue.process_task(request_item, self.download)
+        elif request_item := TaskQueue.DOWNLOAD.pop():
+            if request_item.params.retry > self.spider_max_retries:
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    process_task_type=request_item.__class__.__name__,
+                    retries=request_item.params.retry,
+                    seed_status=DealModel.fail,
+                    **request_item.to_dict
+                )
+                TaskQueue.DONE.push(request_item.seed)
+            else:
+                TaskQueue.process_task(request_item, self.download)
         time.sleep(thread_sleep)
 
     @Decorators.pause
     def build_parse_item(self):
         thread_sleep = 0.1
         if TaskQueue.UPLOAD.length >= self.upload_queue_size:
-            # logger.info(f"upload queue is full, sleep {thread_sleep}s")
             thread_sleep = 5
-        if response_item := TaskQueue.RESPONSE.pop():
-            TaskQueue.process_task(response_item, self.parse)
+        elif response_item := TaskQueue.RESPONSE.pop():
+            if response_item.params.retry > self.spider_max_retries:
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    process_task_type=response_item.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    retries=response_item.params.retry,
+                    **response_item.to_dict
+                )
+                TaskQueue.DONE.push(response_item.seed)
+            else:
+                TaskQueue.process_task(response_item, self.parse)
         time.sleep(thread_sleep)
 
 

{cobweb-launcher-1.3.11 → cobweb-launcher-1.3.13}/cobweb/launchers/launcher.py

@@ -60,6 +60,7 @@ class Launcher(threading.Thread):
         self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
         self.delete_queue_size = setting.DELETE_QUEUE_SIZE
         self.done_queue_size = setting.DONE_QUEUE_SIZE
+        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
 
         self.spider_thread_num = setting.SPIDER_THREAD_NUM
 
@@ -71,7 +72,7 @@ class Launcher(threading.Thread):
         self.crawler_path = setting.CRAWLER
         self.pipeline_path = setting.PIPELINE
 
-        self._threads = []
+        self._thread_info = {}
 
         self._task_info = dict(todo={}, download={})
 
@@ -130,9 +131,12 @@ class Launcher(threading.Thread):
     def check_alive(self):
         while not self.stop.is_set():
             if not self.pause.is_set():
-                for thread in self._threads:
-                    if not thread.is_alive():
-                        thread.start()
+                for name, thread_info in self._thread_info.items():
+                    instance = thread_info['instance']
+                    if not instance.is_alive():
+                        instance = threading.Thread(name=name, target=thread_info['func'], args=())
+                        self._thread_info[name] = dict(instance=instance, func=thread_info['func'])
+                        instance.start()
             time.sleep(1)
 
     def _add_thread(self, func, num=1, obj=None, name=None, args=()):
@@ -140,7 +144,9 @@ class Launcher(threading.Thread):
         name = obj.__class__.__name__ + ":" + (name or func.__name__)
         for i in range(num):
             func_name = name + "_" + str(i) if num > 1 else name
-            self._threads.append(threading.Thread(name=func_name, target=func, args=()))
+            instance = threading.Thread(name=func_name, target=func, args=())
+            self._thread_info[func_name] = dict(instance=instance, func=func)
+            instance.start()
 
     @Decorators.stop
     def _polling(self):
@@ -187,8 +193,8 @@ class Launcher(threading.Thread):
         Crawler = dynamic_load_class(self.crawler_path)
         Pipeline = dynamic_load_class(self.pipeline_path)
 
-        crawler = Crawler(pause=self.pause, custom_func=self.custom_func)
-        pipeline = Pipeline(pause=self.pause)
+        crawler = Crawler(stop=self.stop, pause=self.pause, custom_func=self.custom_func)
+        pipeline = Pipeline(stop=self.stop, pause=self.pause)
 
         self._add_thread(obj=crawler, func=crawler.build_request_item)
         self._add_thread(obj=crawler, func=crawler.build_download_item, num=self.spider_thread_num)

{cobweb-launcher-1.3.11 → cobweb-launcher-1.3.13}/cobweb/launchers/launcher_api.py

@@ -1,6 +1,6 @@
 import time
 
-from cobweb.base import TaskQueue, Decorators
+from cobweb.base import TaskQueue, Decorators, Seed, Request
 from cobweb.schedulers import ApiScheduler
 from .launcher import Launcher
 
@@ -16,13 +16,13 @@ class LauncherApi(Launcher):
     @Decorators.stop
     def _schedule(self):
         thread_sleep = self.scheduling_wait_time
-        for q, key, size, item_info in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"]),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"]),
+        for q, key, size, item_info, Cls in [
+            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"], Seed),
+            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"], Request),
         ]:
             if q.length < size:
                 for member, priority in self._scheduler.schedule(key, self.scheduling_size):
-                    q.push((member, priority), direct_insertion=True)
+                    q.push(Cls(member, priority=priority))
                     self.add_working_item(key.split(":")[-1], member, priority)
                 thread_sleep = 0.1
         time.sleep(thread_sleep)

{cobweb-launcher-1.3.11 → cobweb-launcher-1.3.13}/cobweb/launchers/launcher_pro.py

@@ -1,6 +1,6 @@
 import time
 
-from cobweb.base import TaskQueue, Decorators
+from cobweb.base import TaskQueue, Decorators, Seed, Request
 from cobweb.schedulers import RedisScheduler
 from .launcher import Launcher
 
@@ -16,13 +16,13 @@ class LauncherPro(Launcher):
     @Decorators.stop
     def _schedule(self):
         thread_sleep = self.scheduling_wait_time
-        for q, key, size, item_info in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"]),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"]),
+        for q, key, size, item_info, Cls in [
+            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"], Seed),
+            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"], Request),
         ]:
             if q.length < size:
                 for member, priority in self._scheduler.schedule(key, self.scheduling_size):
-                    q.push((member, priority), direct_insertion=True)
+                    q.push(Cls(member, priority=priority))
                     self.add_working_item(key.split(":")[-1], member, priority)
                 thread_sleep = 0.1
         time.sleep(thread_sleep)

{cobweb-launcher-1.3.11 → cobweb-launcher-1.3.13}/cobweb/pipelines/pipeline.py

@@ -10,9 +10,11 @@ class Pipeline(ABC):
 
     def __init__(
             self,
+            stop: threading.Event,
             pause: threading.Event,
     ):
         super().__init__()
+        self.stop = stop
         self.pause = pause
         self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
         self.upload_wait_time = setting.UPLOAD_WAIT_TIME

{cobweb-launcher-1.3.11 → cobweb-launcher-1.3.13/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.11
+Version: 1.3.13
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.11 → cobweb-launcher-1.3.13}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.3.11",
+    version="1.3.13",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",