cobweb-launcher 1.3.11__py3-none-any.whl → 1.3.13__py3-none-any.whl

cobweb/base/__init__.py

@@ -100,14 +100,15 @@ class Decorators:
     def pause(func):
         @wraps(func)
         def wrapper(self, *args, **kwargs):
-            while not self.pause.is_set():
-                try:
-                    func(self)
-                except Exception as e:
-                    logger.info(f"{func.__name__}: " + str(e))
-                finally:
-                    time.sleep(0.1)
-            logger.info(f"{func.__name__}: close!")
+            while not self.stop.is_set():
+                while not self.pause.is_set():
+                    try:
+                        func(self)
+                    except Exception as e:
+                        logger.info(f"{func.__name__}: " + str(e))
+                    finally:
+                        time.sleep(0.1)
+                # logger.info(f"{func.__name__}: close!")
 
         return wrapper
 

cobweb/base/basic.py

@@ -142,7 +142,7 @@ class Request:
 
     def __init__(
             self,
-            url,
+            # url,
             seed,
             random_ua=True,
             check_status_code=True,
@@ -152,7 +152,7 @@ class Request:
             status=None,
             **kwargs
     ):
-        self.url = url
+        # self.url = url
         self.check_status_code = check_status_code
         self.request_setting = {}
 

cobweb/crawlers/crawler.py

@@ -1,4 +1,5 @@
 import json
+import os
 import time
 import threading
 from typing import Union, Callable, Mapping
@@ -12,15 +13,15 @@ from cobweb.base import (
     ConsoleItem,
     Decorators,
     TaskQueue,
-    logger
 )
 from cobweb.constant import DealModel
 
 
 class Crawler(threading.Thread):
 
-    def __init__(self, pause, custom_func: Union[Mapping[str, Callable]]):
+    def __init__(self, stop, pause, custom_func: Union[Mapping[str, Callable]]):
         super().__init__()
+        self.stop = stop
         self.pause = pause
         for func_name, _callable in custom_func.items():
             if isinstance(_callable, Callable):
@@ -52,14 +53,20 @@ class Crawler(threading.Thread):
         thread_sleep = 0.1
         if TaskQueue.REQUEST.length >= self.request_queue_size:
             thread_sleep = 5
-        elif seed_info := TaskQueue.TODO.pop():
-            member, priority = seed_info
-            seed = Seed(member, priority=priority)
+        elif seed := TaskQueue.TODO.pop():
+            # member, priority = seed_info
+            # seed = Seed(member, priority=priority)
             if seed.params.retry > self.spider_max_retries:
-                seed.params.seed_status = DealModel.fail
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    process_task_type=seed.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    retries=seed.params.retry,
+                    **seed.to_dict
+                )
             else:
                 TaskQueue.process_task(seed, self.request)
-            TaskQueue.DELETE.push(member)
+            TaskQueue.DELETE.push(seed.seed)
         time.sleep(thread_sleep)
 
     @Decorators.pause
@@ -67,22 +74,37 @@ class Crawler(threading.Thread):
         thread_sleep = 0.1
         if TaskQueue.RESPONSE.length >= self.download_queue_size:
             thread_sleep = 5
-            # logger.info(f"download queue is full, sleep {thread_sleep}s")
-        elif request_info := TaskQueue.DOWNLOAD.pop():
-            member, priority = request_info
-            request_setting = json.loads(member)
-            request_item = Request(seed=member, **request_setting)
-            TaskQueue.process_task(request_item, self.download)
+        elif request_item := TaskQueue.DOWNLOAD.pop():
+            if request_item.params.retry > self.spider_max_retries:
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    process_task_type=request_item.__class__.__name__,
+                    retries=request_item.params.retry,
+                    seed_status=DealModel.fail,
+                    **request_item.to_dict
+                )
+                TaskQueue.DONE.push(request_item.seed)
+            else:
+                TaskQueue.process_task(request_item, self.download)
         time.sleep(thread_sleep)
 
     @Decorators.pause
     def build_parse_item(self):
         thread_sleep = 0.1
         if TaskQueue.UPLOAD.length >= self.upload_queue_size:
-            # logger.info(f"upload queue is full, sleep {thread_sleep}s")
             thread_sleep = 5
-        if response_item := TaskQueue.RESPONSE.pop():
-            TaskQueue.process_task(response_item, self.parse)
+        elif response_item := TaskQueue.RESPONSE.pop():
+            if response_item.params.retry > self.spider_max_retries:
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    process_task_type=response_item.__class__.__name__,
+                    seed_status=DealModel.fail,
+                    retries=response_item.params.retry,
+                    **response_item.to_dict
+                )
+                TaskQueue.DONE.push(response_item.seed)
+            else:
+                TaskQueue.process_task(response_item, self.parse)
         time.sleep(thread_sleep)
 
 

cobweb/launchers/launcher.py

@@ -60,6 +60,7 @@ class Launcher(threading.Thread):
         self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
         self.delete_queue_size = setting.DELETE_QUEUE_SIZE
         self.done_queue_size = setting.DONE_QUEUE_SIZE
+        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
 
         self.spider_thread_num = setting.SPIDER_THREAD_NUM
 
@@ -71,7 +72,7 @@ class Launcher(threading.Thread):
         self.crawler_path = setting.CRAWLER
         self.pipeline_path = setting.PIPELINE
 
-        self._threads = []
+        self._thread_info = {}
 
         self._task_info = dict(todo={}, download={})
 
@@ -130,9 +131,12 @@ class Launcher(threading.Thread):
     def check_alive(self):
         while not self.stop.is_set():
             if not self.pause.is_set():
-                for thread in self._threads:
-                    if not thread.is_alive():
-                        thread.start()
+                for name, thread_info in self._thread_info.items():
+                    instance = thread_info['instance']
+                    if not instance.is_alive():
+                        instance = threading.Thread(name=name, target=thread_info['func'], args=())
+                        self._thread_info[name] = dict(instance=instance, func=thread_info['func'])
+                        instance.start()
             time.sleep(1)
 
     def _add_thread(self, func, num=1, obj=None, name=None, args=()):
@@ -140,7 +144,9 @@ class Launcher(threading.Thread):
         name = obj.__class__.__name__ + ":" + (name or func.__name__)
         for i in range(num):
             func_name = name + "_" + str(i) if num > 1 else name
-            self._threads.append(threading.Thread(name=func_name, target=func, args=()))
+            instance = threading.Thread(name=func_name, target=func, args=())
+            self._thread_info[func_name] = dict(instance=instance, func=func)
+            instance.start()
 
     @Decorators.stop
     def _polling(self):
@@ -187,8 +193,8 @@ class Launcher(threading.Thread):
         Crawler = dynamic_load_class(self.crawler_path)
         Pipeline = dynamic_load_class(self.pipeline_path)
 
-        crawler = Crawler(pause=self.pause, custom_func=self.custom_func)
-        pipeline = Pipeline(pause=self.pause)
+        crawler = Crawler(stop=self.stop, pause=self.pause, custom_func=self.custom_func)
+        pipeline = Pipeline(stop=self.stop, pause=self.pause)
 
         self._add_thread(obj=crawler, func=crawler.build_request_item)
         self._add_thread(obj=crawler, func=crawler.build_download_item, num=self.spider_thread_num)

cobweb/launchers/launcher_api.py

@@ -1,6 +1,6 @@
 import time
 
-from cobweb.base import TaskQueue, Decorators
+from cobweb.base import TaskQueue, Decorators, Seed, Request
 from cobweb.schedulers import ApiScheduler
 from .launcher import Launcher
 
@@ -16,13 +16,13 @@ class LauncherApi(Launcher):
     @Decorators.stop
     def _schedule(self):
         thread_sleep = self.scheduling_wait_time
-        for q, key, size, item_info in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"]),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"]),
+        for q, key, size, item_info, Cls in [
+            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"], Seed),
+            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"], Request),
         ]:
             if q.length < size:
                 for member, priority in self._scheduler.schedule(key, self.scheduling_size):
-                    q.push((member, priority), direct_insertion=True)
+                    q.push(Cls(member, priority=priority))
                     self.add_working_item(key.split(":")[-1], member, priority)
                 thread_sleep = 0.1
         time.sleep(thread_sleep)

cobweb/launchers/launcher_pro.py

@@ -1,6 +1,6 @@
 import time
 
-from cobweb.base import TaskQueue, Decorators
+from cobweb.base import TaskQueue, Decorators, Seed, Request
 from cobweb.schedulers import RedisScheduler
 from .launcher import Launcher
 
@@ -16,13 +16,13 @@ class LauncherPro(Launcher):
     @Decorators.stop
     def _schedule(self):
         thread_sleep = self.scheduling_wait_time
-        for q, key, size, item_info in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"]),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"]),
+        for q, key, size, item_info, Cls in [
+            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"], Seed),
+            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"], Request),
         ]:
             if q.length < size:
                 for member, priority in self._scheduler.schedule(key, self.scheduling_size):
-                    q.push((member, priority), direct_insertion=True)
+                    q.push(Cls(member, priority=priority))
                     self.add_working_item(key.split(":")[-1], member, priority)
                 thread_sleep = 0.1
         time.sleep(thread_sleep)

cobweb/pipelines/pipeline.py

@@ -10,9 +10,11 @@ class Pipeline(ABC):
 
     def __init__(
             self,
+            stop: threading.Event,
             pause: threading.Event,
     ):
         super().__init__()
+        self.stop = stop
         self.pause = pause
         self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
         self.upload_wait_time = setting.UPLOAD_WAIT_TIME

{cobweb_launcher-1.3.11.dist-info → cobweb_launcher-1.3.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.11
+Version: 1.3.13
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.3.11.dist-info → cobweb_launcher-1.3.13.dist-info}/RECORD

@@ -1,8 +1,8 @@
 cobweb/__init__.py,sha256=oaEfsGUuGP0s39UbFRwrnsjMUeuB6QvQIAwStKFyUTk,83
 cobweb/constant.py,sha256=eofONAntk9O6S-cb4KbYGYHL_u7nBlOqqFOw_HzJHAU,3588
 cobweb/setting.py,sha256=pY6LKsgWI3164GiGA1z_y26LVf5-3mpiEgmm86mKRdY,3135
-cobweb/base/__init__.py,sha256=T8raRjxGJW27SdRa04KcqvCVlUvRjv1jMLB4wNP_7xc,5169
-cobweb/base/basic.py,sha256=s5G4LBZiLUfoymV-gLSIqeH-OJ7q7-L35sBa6xEH3EI,7666
+cobweb/base/__init__.py,sha256=CgNg7BK8uPICSWrLI9Bi6vNQaquBNY2H31TrDo9-fTI,5245
+cobweb/base/basic.py,sha256=eOSHnZT2xR-sOND8J4M3iCJJJUV51QiFi8Yn8JxV7s4,7670
 cobweb/base/common_queue.py,sha256=Gor7sR3h1hlZWaI0XcNAbf0S15Ftjr3DFRWNTGL13uU,1137
 cobweb/base/dotting.py,sha256=lfFXXqnVP__hxlW3qH5Bnuq69KtnFaQLbcz1M8e2Ajg,1239
 cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
@@ -11,19 +11,19 @@ cobweb/base/request.py,sha256=acGm3OzxsPed5VUTk7D9eeHZPMh7KUNQRUv44G5znZg,2659
 cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=PN5J4gKPEXylwyQeSGOBfauxHktxFr7RJe8nVX1hBw4,2987
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
-cobweb/crawlers/crawler.py,sha256=ZQ6yVA1EaQRdKJEY3DNqShzp9HPMwlSXapnsRW9E5Wc,2987
+cobweb/crawlers/crawler.py,sha256=hmnvQWKC9EdVrn3bGcV4VlwX8KDpVFylCPfuoV307tc,3960
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
 cobweb/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=m_XNG2bWuMbirPt3d0_s-Ezl1xycfUxeqZnwq_kkfuo,116
-cobweb/launchers/launcher.py,sha256=O7HjZ2lewsZ5IIiZ2vhCAqGidJ1JKMxS5fs4KlslBkA,7332
+cobweb/launchers/launcher.py,sha256=KB7aL38T3uMh1s78HyHLcS0DS8ovx7VuW6JHn5ooec8,7807
 cobweb/launchers/launcher_air.py,sha256=yPr395HVIIHAq6lqRcYJu7c0KkfO9V8O-2sn0hC96p0,2990
-cobweb/launchers/launcher_api.py,sha256=TfLrLXazFWsOJLI7caMGfZozCttL1WTwTo3uUpN_FV0,3370
-cobweb/launchers/launcher_pro.py,sha256=2H-TcvQx-ga78GLNTa-GXMLYAj9nEeCJSWf8xl-1ISQ,3374
+cobweb/launchers/launcher_api.py,sha256=vpwFxB1azgBk1bS7VhX3jOprQS8fl6Iu_5M-Y3QT67A,3394
+cobweb/launchers/launcher_pro.py,sha256=Kl64CQLcUwW9FeaKFg0GIK51fLZaWkoQlQ346zEvJEE,3398
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
-cobweb/pipelines/pipeline.py,sha256=Pycm22bHId9a3gdP81D5y7SsuMndYooTb5n4zQxP7dM,1321
+cobweb/pipelines/pipeline.py,sha256=FHY7ZHNZgx-AdbHt1MKHm-w0aigxvyXFV10T0NMdrOE,1381
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
 cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
 cobweb/schedulers/__init__.py,sha256=y7Lv_7b0zfTl0OhIONb_8u1K1C9gVlBA-xz_XG_kI9g,85
@@ -33,8 +33,8 @@ cobweb/utils/__init__.py,sha256=YvD4mIDBd9jmGA6WJBcwkgDU2jRFNBCEbarZCSUBAHE,114
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/oss.py,sha256=6Qlhdde7CcwD69bBe2rGWHY3-aptG9NXB_DZLhjgDRQ,3553
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.3.11.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.3.11.dist-info/METADATA,sha256=5GKTN8jXVIcQY-kAcbc2-o8NPPImj_9KFrwaBXKhXMY,6510
-cobweb_launcher-1.3.11.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.3.11.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.3.11.dist-info/RECORD,,
+cobweb_launcher-1.3.13.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.3.13.dist-info/METADATA,sha256=_2gVLBf6ZRO9NA1zwD4kLFHKkbLFo8uEP7EYL7WdjaQ,6510
+cobweb_launcher-1.3.13.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.3.13.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.3.13.dist-info/RECORD,,