cobweb-launcher 1.3.15__py3-none-any.whl → 3.1.0__py3-none-any.whl

cobweb/launchers/distributor.py

@@ -0,0 +1,171 @@
+import time
+import threading
+import traceback
+from inspect import isgenerator
+from typing import Callable
+from urllib.parse import urlparse
+from requests import Response as Res
+
+from cobweb import setting
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.base import (
+    Seed,
+    Queue,
+    BaseItem,
+    Request,
+    Response,
+    logger
+)
+from cobweb.utils import LoghubDot, check_pause
+
+
+class Distributor(threading.Thread):
+
+    def __init__(
+            self,
+            task: str,
+            project: str,
+            new: Queue,
+            todo: Queue,
+            done: Queue,
+            upload: Queue,
+            register: Callable,
+            stop: threading.Event,
+            pause: threading.Event,
+            SpiderCrawler
+    ):
+        super().__init__()
+        self.task = task
+        self.project = project
+        self.stop = stop
+        self.pause = pause
+
+        self.new = new
+        self.todo = todo
+        self.done = done
+        self.upload = upload
+        self.register = register
+
+        self.time_sleep = setting.SPIDER_TIME_SLEEP
+        self.thread_num = setting.SPIDER_THREAD_NUM
+        self.max_retries = setting.SPIDER_MAX_RETRIES
+        self.record_failed = setting.RECORD_FAILED_SPIDER
+        self.loghub_dot = LoghubDot()  # todo: 解偶
+
+        self.Crawler = SpiderCrawler
+
+        logger.debug(f"Distribute instance attrs: {self.__dict__}")
+
+    def distribute(self, item, seed, _id: int):
+        if isinstance(item, Request):
+            seed.params.start_time = time.time()
+            self.process(item=seed, seed=seed, callback=self.Crawler.download, _id=1)
+        elif isinstance(item, Response):
+            if _id == 2:
+                raise TypeError("parse function can't yield a Response instance")
+            dot = isinstance(item.response, Res)
+            self.spider_logging(seed, item, dot=dot)
+            self.process(item=seed, seed=seed, callback=self.Crawler.parse, _id=2)
+        elif isinstance(item, BaseItem):
+            self.upload.push(item)
+        elif isinstance(item, Seed):
+            self.new.push((seed, item), direct_insertion=True)
+        elif isinstance(item, str) and item == DealModel.poll:
+            self.todo.push(seed)
+        elif isinstance(item, str) and item == DealModel.done:
+            self.done.push(seed)
+        elif isinstance(item, str) and item == DealModel.fail:
+            seed.params.retry += 1
+            if seed.params.retry < self.max_retries:
+                self.todo.push(seed)
+            else:
+                if record_failed := self.record_failed:
+                    try:
+                        response = Response(seed, "failed", max_retries=True)
+                        self.process(response, seed, self.Crawler.parse, _id=2)
+                    except:
+                        record_failed = False
+                if not record_failed:
+                    self.done.push(seed)
+        else:
+            raise TypeError("yield value type error!")
+
+    def process(self, item, seed, callback, _id: int):
+        result_iterators = callback(item)
+        if not isgenerator(result_iterators):
+            raise TypeError(f"{callback.__name__} function isn't a generator!")
+        for result_item in result_iterators:
+            self.distribute(result_item, seed, _id)
+
+    @check_pause
+    def spider(self):
+        if seed := self.todo.pop():
+            try:
+                self.process(item=seed, seed=seed, callback=self.Crawler.request, _id=0)
+            except Exception as e:
+                url, status = seed.url, e.__class__.__name__
+                msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                if getattr(e, "response", None) and isinstance(e.response, Res):
+                    url = e.response.request.url
+                    status = e.response.status_code
+                self.spider_logging(seed, None, error=True, url=url, status=status, msg=msg)
+                self.distribute(DealModel.fail, seed, _id=-1)
+
+    def spider_logging(
+            self, seed,
+            item: Response = None,
+            error: bool = False,
+            dot: bool = True,
+            **kwargs
+    ):
+        detail_log_info = LogTemplate.log_info(seed.to_dict)
+        if error:
+            url = kwargs.get("url")
+            msg = kwargs.get("msg")
+            status = kwargs.get("status")
+            if dot:
+                self.loghub_dot.build(
+                    topic=urlparse(url).netloc,
+                    data_size=-1, cost_time=-1,
+                    status=status, url=url,
+                    seed=seed.to_string,
+                    proxy_type=seed.params.proxy_type,
+                    proxy=seed.params.proxy,
+                    project=self.project,
+                    task=self.task, msg=msg,
+                )
+            logger.info(LogTemplate.download_exception.format(
+                detail=detail_log_info,
+                retry=seed.params.retry,
+                priority=seed.params.priority,
+                seed_version=seed.params.seed_version,
+                identifier=seed.identifier or "",
+                exception=msg
+            ))
+        else:
+            logger.info(LogTemplate.download_info.format(
+                detail=detail_log_info,
+                retry=seed.params.retry,
+                priority=seed.params.priority,
+                seed_version=seed.params.seed_version,
+                identifier=seed.identifier or "",
+                status=item.response,
+                response=LogTemplate.log_info(item.to_dict)
+            ))
+            if dot:
+                end_time = time.time()
+                stime = seed.params.start_time
+                cost_time = end_time - stime if stime else -1
+                topic = urlparse(item.response.request.url).netloc
+                data_size = int(item.response.headers.get("content-length", 0))
+                self.loghub_dot.build(
+                    topic=topic, data_size=data_size, cost_time=cost_time,
+                    status=200, seed=seed.to_string, url=item.response.url,
+                    proxy=seed.params.proxy, proxy_type=seed.params.proxy_type,
+                    project=self.project, task=self.task,
+                )
+
+    def run(self):
+        self.register(self.loghub_dot.build_run, tag="LoghubDot")
+        for _ in range(self.thread_num):
+            self.register(self.spider, tag="Distributor")

cobweb/launchers/launcher.py

@@ -1,26 +1,34 @@
 import time
+import uuid
 import inspect
 import threading
 import importlib
 
-from cobweb.constant import LogTemplate
-from cobweb.utils import dynamic_load_class
-from cobweb.base import TaskQueue, Decorators, logger
 from cobweb import setting
+from cobweb.base import Seed, Queue, logger
+from cobweb.utils.tools import dynamic_load_class
+from cobweb.launchers import Distributor, Uploader
 
 
-class Launcher(threading.Thread):
+class Launcher:
 
-    __CUSTOM_FUNC__ = {}
+    _NEW_QUEUE_ = Queue()
+    _TODO_QUEUE_ = Queue()
+    _DONE_QUEUE_ = Queue()
+    _UPLOAD_QUEUE_ = Queue()
+
+    __WORKER_THREAD__ = dict()
+    __REGISTER_FUNC__ = dict()
 
     def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__()
+
         self.task = task
         self.project = project
-        self.custom_func = dict()
-        self.app_time = int(time.time())
 
-        self.check_emtpy_times = 0
+        self._app_time = int(time.time())
+        self._stop = threading.Event()  # 结束事件
+        self._pause = threading.Event()  # 暂停事件
 
         _setting = dict()
 
@@ -41,40 +49,12 @@ class Launcher(threading.Thread):
         for k, v in _setting.items():
             setattr(setting, k.upper(), v)
 
-        self.before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
-
-        self.scheduling_wait_time = setting.SCHEDULING_WAIT_TIME
-        self.inserting_wait_time = setting.INSERTING_WAIT_TIME
-        self.removing_wait_time = setting.REMOVING_WAIT_TIME
-        self.seed_reset_seconds = setting.SEED_RESET_SECONDS
-
-        self.scheduling_size = setting.SCHEDULING_SIZE
-        self.inserting_size = setting.INSERTING_SIZE
-        self.removing_size = setting.REMOVING_SIZE
-
-        self.todo_queue_size = setting.TODO_QUEUE_SIZE
-        self.seed_queue_size = setting.SEED_QUEUE_SIZE
-        self.request_queue_size = setting.REQUEST_QUEUE_SIZE
-        self.download_queue_size = setting.DOWNLOAD_QUEUE_SIZE
-        self.response_queue_size = setting.RESPONSE_QUEUE_SIZE
-        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
-        self.delete_queue_size = setting.DELETE_QUEUE_SIZE
-        self.done_queue_size = setting.DONE_QUEUE_SIZE
-        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
-
-        self.spider_thread_num = setting.SPIDER_THREAD_NUM
+        self._done_model = setting.DONE_MODEL
+        self._task_model = setting.TASK_MODEL
 
-        self.task_model = setting.TASK_MODEL
-
-        self.stop = threading.Event()  # 结束事件
-        self.pause = threading.Event()  # 暂停事件
-
-        self.crawler_path = setting.CRAWLER
-        self.pipeline_path = setting.PIPELINE
-
-        self._thread_info = {}
-
-        self._task_info = dict(todo={}, download={})
+        self.Scheduler = dynamic_load_class(setting.SCHEDULER)
+        self.SpiderCrawler = dynamic_load_class(setting.CRAWLER)
+        self.SpiderPipeline = dynamic_load_class(setting.PIPELINE)
 
     @property
     def request(self):
@@ -88,7 +68,7 @@ class Launcher(threading.Thread):
                 yield Request(seed.url, seed)
         """
         def decorator(func):
-            self.custom_func['request'] = func
+            self.SpiderCrawler.request = func
         return decorator
 
     @property
@@ -103,7 +83,7 @@ class Launcher(threading.Thread):
                 yield Response(item.seed, response)
         """
         def decorator(func):
-            self.custom_func['download'] = func
+            self.SpiderCrawler.download = func
         return decorator
 
     @property
@@ -118,94 +98,70 @@ class Launcher(threading.Thread):
                yield xxxItem(seed, **kwargs)
         """
         def decorator(func):
-            self.custom_func['parse'] = func
+            self.SpiderCrawler.parse = func
         return decorator
 
-    def remove_working_items(self, key, items):
-        for item in items:
-            self._task_info[key].pop(item, None)
-
-    def add_working_item(self, key, member, priority):
-        self._task_info[key][member] = priority
-
-    def check_alive(self):
-        while not self.stop.is_set():
-            if not self.pause.is_set():
-                for name, thread_info in self._thread_info.items():
-                    instance = thread_info['instance']
-                    if not instance.is_alive():
-                        instance = threading.Thread(name=name, target=thread_info['func'], args=())
-                        self._thread_info[name] = dict(instance=instance, func=thread_info['func'])
-                        instance.start()
-            time.sleep(1)
-
-    def _add_thread(self, func, num=1, obj=None, name=None, args=()):
-        obj = obj or self
-        name = obj.__class__.__name__ + ":" + (name or func.__name__)
-        for i in range(num):
-            func_name = name + "_" + str(i) if num > 1 else name
-            instance = threading.Thread(name=func_name, target=func, args=())
-            self._thread_info[func_name] = dict(instance=instance, func=func)
-            instance.start()
-
-    @Decorators.stop
-    def _polling(self):
-        time.sleep(10)
-        if self.pause.is_set():
-            run_time = int(time.time()) - self.app_time
-            if not self.task_model and run_time > self.before_scheduler_wait_seconds:
-                logger.info("Done! ready to close thread...")
-                self.stop.set()
-            elif TaskQueue.TODO.length or TaskQueue.DOWNLOAD.length:
-                logger.info(f"Recovery {self.task} task run！")
-                self.check_emtpy_times = 0
-                self.pause.clear()
-            else:
-                logger.info("pause! waiting for resume...")
-        elif TaskQueue.is_empty() and self.check_emtpy_times > 2:
-            logger.info("pause! waiting for resume...")
-            self.doing_seeds = {}
-            self._task_info['todo'] = {}
-            self._task_info['download'] = {}
-            self.pause.set()
-        elif TaskQueue.is_empty():
-            logger.info(
-                "check whether the task is complete, "
-                f"reset times {3 - self.check_emtpy_times}"
-            )
-            self.check_emtpy_times += 1
-        else:
-            logger.info(LogTemplate.launcher_polling.format(
-                task=self.task,
-                memory_todo_count=len(self._task_info["todo"]),
-                memory_download_count=len(self._task_info["download"]),
-                todo_queue_len=TaskQueue.TODO.length,
-                delete_queue_len=TaskQueue.DELETE.length,
-                request_queue_len=TaskQueue.REQUEST.length,
-                response_queue_len=TaskQueue.RESPONSE.length,
-                done_queue_len=TaskQueue.DONE.length,
-                upload_queue_len=TaskQueue.UPLOAD.length,
-                seed_queue_len=TaskQueue.SEED.length,
-                download_queue_len=TaskQueue.DOWNLOAD.length
-            ))
-
-    def run(self):
-        Crawler = dynamic_load_class(self.crawler_path)
-        Pipeline = dynamic_load_class(self.pipeline_path)
-
-        crawler = Crawler(stop=self.stop, pause=self.pause, custom_func=self.custom_func)
-        pipeline = Pipeline(stop=self.stop, pause=self.pause)
-
-        self._add_thread(obj=crawler, func=crawler.build_request_item)
-        self._add_thread(obj=crawler, func=crawler.build_download_item, num=self.spider_thread_num)
-        self._add_thread(obj=crawler, func=crawler.build_parse_item)
-        self._add_thread(obj=pipeline, func=pipeline.run)
-
-        self._add_thread(func=self._polling)
-
-        self._init_schedule_thread()
-        self.check_alive()
-
-    def _init_schedule_thread(self):
-        ...
+    def start_seeds(self, seeds: list):
+        seed_list = [Seed(seed) for seed in seeds]
+        self._TODO_QUEUE_.push(seed_list)
+        return seed_list
+
+    def _register(self, func, tag: str = "launcher"):
+        name = tag + ":" + func.__name__ + "_" + str(uuid.uuid4())
+        self.__REGISTER_FUNC__[name] = func
+        if not self.__WORKER_THREAD__.get(name):
+            worker_thread = threading.Thread(name=name, target=func)
+            self.__WORKER_THREAD__[name] = worker_thread
+            worker_thread.start()
+
+    def _monitor(self):
+        while not self._stop.is_set():
+            if self._pause.is_set():
+                time.sleep(15)
+                continue
+            for name, worker_thread in self.__WORKER_THREAD__.items():
+                if not worker_thread.is_alive():
+                    logger.info(f"{name} thread is dead. Restarting...")
+                    func = self.__REGISTER_FUNC__[name]
+                    worker_thread = threading.Thread(name=name, target=func)
+                    self.__WORKER_THREAD__[name] = worker_thread
+                    worker_thread.start()
+                time.sleep(3)
+
+    def start(self):
+        self._pause.is_set()
+
+        self.Scheduler(
+            task=self.task,
+            project=self.project,
+            stop=self._stop,
+            pause=self._pause,
+            new=self._NEW_QUEUE_,
+            todo=self._TODO_QUEUE_,
+            done=self._DONE_QUEUE_,
+            upload=self._UPLOAD_QUEUE_,
+            register=self._register
+        ).start()
+
+        Distributor(
+            task=self.task,
+            project=self.project,
+            new=self._NEW_QUEUE_,
+            todo=self._TODO_QUEUE_,
+            done=self._DONE_QUEUE_,
+            upload=self._UPLOAD_QUEUE_,
+            register=self._register,
+            stop=self._stop, pause=self._pause,
+            SpiderCrawler=self.SpiderCrawler
+        ).start()
+
+        Uploader(
+            stop=self._stop, pause=self._pause,
+            done=self._DONE_QUEUE_,
+            upload=self._UPLOAD_QUEUE_,
+            register=self._register,
+            SpiderPipeline=self.SpiderPipeline
+        ).start()
+
+        self._monitor()
 

cobweb/launchers/uploader.py

@@ -0,0 +1,65 @@
+import time
+import threading
+from typing import Callable
+from cobweb import setting
+from cobweb.base import Queue, logger
+from cobweb.utils import check_pause
+
+
+class Uploader(threading.Thread):
+
+    def __init__(
+            self,
+            stop: threading.Event,
+            pause: threading.Event,
+            upload: Queue, done: Queue,
+            register: Callable,
+            SpiderPipeline
+    ):
+        super().__init__()
+        self.stop = stop
+        self.pause = pause
+
+        self.done = done
+        self.upload = upload
+        self.register = register
+
+        self.upload_size = setting.UPLOAD_QUEUE_MAX_SIZE
+        self.wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
+
+        self.Pipeline = SpiderPipeline
+
+        logger.debug(f"Uploader instance attrs: {self.__dict__}")
+
+    @check_pause
+    def upload(self):
+        if not self.upload.length:
+            time.sleep(self.wait_seconds)
+            return
+        if self.upload.length < self.upload_size:
+            time.sleep(self.wait_seconds)
+        data_info, seeds = {}, []
+        try:
+            for _ in range(self.upload_size):
+                item = self.upload.pop()
+                if not item:
+                    break
+                seeds.append(item.seed)
+                data = self.Pipeline.build(item)
+                data_info.setdefault(item.table, []).append(data)
+            for table, datas in data_info.items():
+                try:
+                    self.Pipeline.upload(table, datas)
+                except Exception as e:
+                    logger.info(e)
+        except Exception as e:
+            logger.info(e)
+        if seeds:
+            self.done.push(seeds)
+
+        logger.info("upload pipeline close!")
+
+    def run(self):
+        self.register(self.upload, tag="Uploader")
+
+

cobweb/pipelines/pipeline.py

@@ -1,48 +1,15 @@
-import time
-import threading
-
 from abc import ABC, abstractmethod
-from cobweb.base import BaseItem, TaskQueue, logger, Decorators
-from cobweb import setting
+from cobweb.base import BaseItem
 
 
 class Pipeline(ABC):
 
-    def __init__(
-            self,
-            stop: threading.Event,
-            pause: threading.Event,
-    ):
-        super().__init__()
-        self.stop = stop
-        self.pause = pause
-        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
-        self.upload_wait_time = setting.UPLOAD_WAIT_TIME
-
     @abstractmethod
     def build(self, item: BaseItem) -> dict:
-        ...
+        pass
 
     @abstractmethod
     def upload(self, table: str, data: list) -> bool:
-        ...
+        pass
 
-    @Decorators.pause
-    def run(self):
-        data_info, seeds = {}, []
-        thread_sleep = self.upload_wait_time if TaskQueue.UPLOAD.length < self.upload_queue_size else 0.1
-        try:
-            while (item := TaskQueue.UPLOAD.pop()) and len(seeds) <= self.upload_queue_size:
-                data = self.build(item)
-                data_info.setdefault(item.table, []).append(data)
-                seeds.append(item.seed)
-            for table, datas in data_info.items():
-                self.upload(table, datas)
-        except Exception as e:
-            logger.info(e)
-            seeds = None
-            # todo: retry
-        finally:
-            TaskQueue.DONE.push(seeds)
 
-        time.sleep(thread_sleep)

cobweb/schedulers/__init__.py

@@ -1,3 +1 @@
-from .scheduler_redis import RedisScheduler
-from .scheduler_api import ApiScheduler
-
+from .scheduler_with_redis import RedisScheduler

cobweb/schedulers/launcher_air.py

@@ -0,0 +1,93 @@
+import time
+
+from cobweb.base import logger
+from cobweb.constant import LogTemplate
+from .launcher import Launcher, check_pause
+
+
+class LauncherAir(Launcher):
+
+    # def _scheduler(self):
+    #     if self.start_seeds:
+    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
+
+    @check_pause
+    def _insert(self):
+        new_seeds = {}
+        del_seeds = set()
+        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+        for _ in range(self._new_queue_max_size):
+            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed_tuple:
+                break
+            seed, new_seed = seed_tuple
+            new_seeds[new_seed.to_string] = new_seed.params.priority
+            del_seeds.add(seed.to_string)
+        if new_seeds:
+            self.__LAUNCHER_QUEUE__['todo'].push(new_seeds)
+        if del_seeds:
+            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
+        if status:
+            time.sleep(self._new_queue_wait_seconds)
+
+    @check_pause
+    def _delete(self):
+        seeds = []
+        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+
+        for _ in range(self._done_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['done'].pop()
+            if not seed:
+                break
+            seeds.append(seed.to_string)
+
+        if seeds:
+            self._remove_doing_seeds(seeds)
+
+        if status:
+            time.sleep(self._done_queue_wait_seconds)
+
+    def _polling(self):
+
+        check_emtpy_times = 0
+
+        while not self._stop.is_set():
+
+            queue_not_empty_count = 0
+            pooling_wait_seconds = 30
+
+            for q in self.__LAUNCHER_QUEUE__.values():
+                if q.length != 0:
+                    queue_not_empty_count += 1
+
+            if queue_not_empty_count == 0:
+                pooling_wait_seconds = 3
+                if self._pause.is_set():
+                    check_emtpy_times = 0
+                    if not self._task_model:
+                        logger.info("Done! Ready to close thread...")
+                        self._stop.set()
+                elif check_emtpy_times > 2:
+                    self.__DOING__ = {}
+                    self._pause.set()
+                else:
+                    logger.info(
+                        "check whether the task is complete, "
+                        f"reset times {3 - check_emtpy_times}"
+                    )
+                    check_emtpy_times += 1
+            elif self._pause.is_set():
+                self._pause.clear()
+                self._execute()
+            else:
+                logger.info(LogTemplate.launcher_air_polling.format(
+                    task=self.task,
+                    doing_len=len(self.__DOING__.keys()),
+                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
+                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
+                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
+                ))
+
+            time.sleep(pooling_wait_seconds)
+
+