cobweb-launcher 1.3.12__tar.gz → 1.3.14__tar.gz

{cobweb-launcher-1.3.12/cobweb_launcher.egg-info → cobweb-launcher-1.3.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.12
+Version: 1.3.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.12 → cobweb-launcher-1.3.14}/cobweb/base/__init__.py

@@ -100,14 +100,15 @@ class Decorators:
     def pause(func):
         @wraps(func)
         def wrapper(self, *args, **kwargs):
-            while not self.pause.is_set():
-                try:
-                    func(self)
-                except Exception as e:
-                    logger.info(f"{func.__name__}: " + str(e))
-                finally:
-                    time.sleep(0.1)
-            logger.info(f"{func.__name__}: close!")
+            while not self.stop.is_set():
+                while not self.pause.is_set():
+                    try:
+                        func(self)
+                    except Exception as e:
+                        logger.info(f"{func.__name__}: " + str(e))
+                    finally:
+                        time.sleep(0.1)
+                # logger.info(f"{func.__name__}: close!")
 
         return wrapper
 

{cobweb-launcher-1.3.12 → cobweb-launcher-1.3.14}/cobweb/base/basic.py

@@ -142,7 +142,7 @@ class Request:
 
     def __init__(
             self,
-            url,
+            # url,
             seed,
             random_ua=True,
             check_status_code=True,
@@ -152,7 +152,7 @@ class Request:
             status=None,
             **kwargs
     ):
-        self.url = url
+        # self.url = url
         self.check_status_code = check_status_code
         self.request_setting = {}
 

{cobweb-launcher-1.3.12 → cobweb-launcher-1.3.14}/cobweb/crawlers/crawler.py

@@ -13,15 +13,15 @@ from cobweb.base import (
     ConsoleItem,
     Decorators,
     TaskQueue,
-    logger
 )
 from cobweb.constant import DealModel
 
 
 class Crawler(threading.Thread):
 
-    def __init__(self, pause, custom_func: Union[Mapping[str, Callable]]):
+    def __init__(self, stop, pause, custom_func: Union[Mapping[str, Callable]]):
         super().__init__()
+        self.stop = stop
         self.pause = pause
         for func_name, _callable in custom_func.items():
             if isinstance(_callable, Callable):
@@ -34,7 +34,7 @@ class Crawler(threading.Thread):
 
     @staticmethod
     def request(seed: Seed) -> Union[Request, BaseItem]:
-        yield Request(seed.url, seed, timeout=5)
+        yield Request(seed, timeout=5)
 
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
@@ -57,12 +57,11 @@ class Crawler(threading.Thread):
             # member, priority = seed_info
             # seed = Seed(member, priority=priority)
             if seed.params.retry > self.spider_max_retries:
-                # seed.params.seed_status = DealModel.fail
                 TaskQueue.DOT.build(
                     topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
-                    # cost_time=round(time.time() - start_time, 2),
                     process_task_type=seed.__class__.__name__,
                     seed_status=DealModel.fail,
+                    retries=seed.params.retry,
                     **seed.to_dict
                 )
             else:
@@ -75,17 +74,12 @@ class Crawler(threading.Thread):
         thread_sleep = 0.1
         if TaskQueue.RESPONSE.length >= self.download_queue_size:
             thread_sleep = 5
-            # logger.info(f"download queue is full, sleep {thread_sleep}s")
         elif request_item := TaskQueue.DOWNLOAD.pop():
-            # member, priority = request_info
-            #
-            # request_setting = json.loads(member)
-            # request_item = Request(seed=member, **request_setting)
             if request_item.params.retry > self.spider_max_retries:
                 TaskQueue.DOT.build(
                     topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
-                    # cost_time=round(time.time() - start_time, 2),
                     process_task_type=request_item.__class__.__name__,
+                    retries=request_item.params.retry,
                     seed_status=DealModel.fail,
                     **request_item.to_dict
                 )
@@ -105,6 +99,7 @@ class Crawler(threading.Thread):
                     topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
                     process_task_type=response_item.__class__.__name__,
                     seed_status=DealModel.fail,
+                    retries=response_item.params.retry,
                     **response_item.to_dict
                 )
                 TaskQueue.DONE.push(response_item.seed)

{cobweb-launcher-1.3.12 → cobweb-launcher-1.3.14}/cobweb/launchers/launcher.py

@@ -72,7 +72,7 @@ class Launcher(threading.Thread):
         self.crawler_path = setting.CRAWLER
         self.pipeline_path = setting.PIPELINE
 
-        self._threads = []
+        self._thread_info = {}
 
         self._task_info = dict(todo={}, download={})
 
@@ -131,9 +131,12 @@ class Launcher(threading.Thread):
     def check_alive(self):
         while not self.stop.is_set():
             if not self.pause.is_set():
-                for thread in self._threads:
-                    if not thread.is_alive():
-                        thread.start()
+                for name, thread_info in self._thread_info.items():
+                    instance = thread_info['instance']
+                    if not instance.is_alive():
+                        instance = threading.Thread(name=name, target=thread_info['func'], args=())
+                        self._thread_info[name] = dict(instance=instance, func=thread_info['func'])
+                        instance.start()
             time.sleep(1)
 
     def _add_thread(self, func, num=1, obj=None, name=None, args=()):
@@ -141,7 +144,9 @@ class Launcher(threading.Thread):
         name = obj.__class__.__name__ + ":" + (name or func.__name__)
         for i in range(num):
             func_name = name + "_" + str(i) if num > 1 else name
-            self._threads.append(threading.Thread(name=func_name, target=func, args=()))
+            instance = threading.Thread(name=func_name, target=func, args=())
+            self._thread_info[func_name] = dict(instance=instance, func=func)
+            instance.start()
 
     @Decorators.stop
     def _polling(self):
@@ -188,8 +193,8 @@ class Launcher(threading.Thread):
         Crawler = dynamic_load_class(self.crawler_path)
         Pipeline = dynamic_load_class(self.pipeline_path)
 
-        crawler = Crawler(pause=self.pause, custom_func=self.custom_func)
-        pipeline = Pipeline(pause=self.pause)
+        crawler = Crawler(stop=self.stop, pause=self.pause, custom_func=self.custom_func)
+        pipeline = Pipeline(stop=self.stop, pause=self.pause)
 
         self._add_thread(obj=crawler, func=crawler.build_request_item)
         self._add_thread(obj=crawler, func=crawler.build_download_item, num=self.spider_thread_num)

{cobweb-launcher-1.3.12 → cobweb-launcher-1.3.14}/cobweb/launchers/launcher_pro.py

@@ -1,6 +1,6 @@
 import time
 
-from cobweb.base import TaskQueue, Decorators
+from cobweb.base import TaskQueue, Decorators, Seed, Request
 from cobweb.schedulers import RedisScheduler
 from .launcher import Launcher
 
@@ -16,13 +16,13 @@ class LauncherPro(Launcher):
     @Decorators.stop
     def _schedule(self):
         thread_sleep = self.scheduling_wait_time
-        for q, key, size, item_info in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"]),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"]),
+        for q, key, size, item_info, Cls in [
+            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"], Seed),
+            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"], Request),
         ]:
             if q.length < size:
                 for member, priority in self._scheduler.schedule(key, self.scheduling_size):
-                    q.push((member, priority), direct_insertion=True)
+                    q.push(Cls(member, priority=priority))
                     self.add_working_item(key.split(":")[-1], member, priority)
                 thread_sleep = 0.1
         time.sleep(thread_sleep)

{cobweb-launcher-1.3.12 → cobweb-launcher-1.3.14}/cobweb/pipelines/pipeline.py

@@ -10,9 +10,11 @@ class Pipeline(ABC):
 
     def __init__(
             self,
+            stop: threading.Event,
             pause: threading.Event,
     ):
         super().__init__()
+        self.stop = stop
         self.pause = pause
         self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
         self.upload_wait_time = setting.UPLOAD_WAIT_TIME

{cobweb-launcher-1.3.12 → cobweb-launcher-1.3.14/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.12
+Version: 1.3.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.12 → cobweb-launcher-1.3.14}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.3.12",
+    version="1.3.14",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",