cobweb-launcher 1.2.49__py3-none-any.whl → 1.3.2__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -1,66 +1,35 @@
 import json
-import os
-import threading
 import time
-import traceback
-from inspect import isgenerator
+import threading
 from typing import Union, Callable, Mapping
-from urllib.parse import urlparse
-
-from requests import Response as Res
 
-from cobweb.constant import DealModel, LogTemplate
+import setting
 from cobweb.base import (
     Seed,
-    BaseItem, 
-    Request, 
-    Response, 
+    BaseItem,
+    Request,
+    Response,
     ConsoleItem,
+    Decorators,
+    TaskQueue,
     logger
 )
-from cobweb.utils import LoghubDot
-proxy_type = os.getenv("PROXY_TYPE", "")
+from constant import DealModel
 
 
 class Crawler(threading.Thread):
 
-    def __init__(
-            self,
-            task: str,
-            project: str,
-            stop: threading.Event,
-            pause: threading.Event,
-            # launcher_queue: Union[Mapping[str, Queue]],
-            get_seed: Callable,
-            set_seed: Callable,
-            add_seed: Callable,
-            delete_seed: Callable,
-            upload_data: Callable,
-            custom_func: Union[Mapping[str, Callable]],
-            thread_num: int,
-            max_retries: int,
-            time_sleep: int,
-    ):
+    def __init__(self, pause, custom_func: Union[Mapping[str, Callable]]):
         super().__init__()
-        self.task = task
-        self.project = project
-        self._stop = stop
-        self._pause = pause
-        self._get_seed = get_seed
-        self._set_seed = set_seed
-        self._add_seed = add_seed
-        self._delete_seed = delete_seed
-        self._upload_data = upload_data
-
+        self.pause = pause
         for func_name, _callable in custom_func.items():
             if isinstance(_callable, Callable):
                 self.__setattr__(func_name, _callable)
 
-        self.thread_num = thread_num
-        self.time_sleep = time_sleep
-        self.max_retries = max_retries
-
-        self.loghub_dot = LoghubDot()
+        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
+        self.request_queue_size = setting.REQUEST_QUEUE_SIZE
+        self.download_queue_size = setting.DOWNLOAD_QUEUE_SIZE
+        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
 
     @staticmethod
     def request(seed: Seed) -> Union[Request, BaseItem]:
@@ -77,142 +46,43 @@ class Crawler(threading.Thread):
         upload_item["text"] = item.response.text
         yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
 
-    # def get_seed(self) -> Seed:
-    #     return self._todo.pop()
-
-    def distribute(self, item, seed):
-        if isinstance(item, BaseItem):
-            self._upload_data(item)
-        elif isinstance(item, Seed):
-            self._add_seed(item)
-        elif isinstance(item, str) and item == DealModel.poll:
-            self._set_seed(seed)
-        elif isinstance(item, str) and item == DealModel.done:
-            self._delete_seed(seed)
-        elif isinstance(item, str) and item == DealModel.fail:
-            seed.params.seed_status = DealModel.fail
-            self._delete_seed(seed)
-        else:
-            raise TypeError("yield value type error!")
-
-    def spider(self):
-        while not self._stop.is_set():
-
-            seed = self._get_seed()
-
-            if not seed:
-                time.sleep(1)
-                continue
-
-            elif seed.params.retry > self.max_retries:
+    # @decorators.add_thread()
+    @Decorators.pause
+    def build_request_item(self):
+        thread_sleep = 0.1
+        if TaskQueue.REQUEST.length >= self.request_queue_size:
+            thread_sleep = 5
+        elif seed_info := TaskQueue.TODO.pop():
+            member, priority = seed_info
+            seed = Seed(member, priority=priority)
+            if seed.params.retry > self.spider_max_retries:
                 seed.params.seed_status = DealModel.fail
-                self._delete_seed(seed)
-                continue
-
-            seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
-
-            try:
-                request_iterators = self.request(seed)
-
-                if not isgenerator(request_iterators):
-                    raise TypeError("request function isn't a generator!")
-
-                iterator_status = False
-
-                for request_item in request_iterators:
-
-                    iterator_status = True
-
-                    if isinstance(request_item, Request):
-                        iterator_status = False
-                        start_time = time.time()
-                        download_iterators = self.download(request_item)
-                        if not isgenerator(download_iterators):
-                            raise TypeError("download function isn't a generator")
-
-                        for download_item in download_iterators:
-                            iterator_status = True
-                            if isinstance(download_item, Response):
-                                iterator_status = False
-                                logger.info(LogTemplate.download_info.format(
-                                    detail=seed_detail_log_info,
-                                    retry=seed.params.retry,
-                                    priority=seed.params.priority,
-                                    seed_version=seed.params.seed_version,
-                                    identifier=seed.identifier or "",
-                                    status=download_item.response,
-                                    response=LogTemplate.log_info(download_item.to_dict)
-                                ))
-                                if isinstance(download_item.response, Res):
-                                    end_time = time.time()
-                                    self.loghub_dot.build(
-                                        topic=urlparse(download_item.response.request.url).netloc,
-                                        data_size=int(download_item.response.headers.get("content-length", 0)),
-                                        cost_time=end_time - start_time, status = 200,
-                                        url=download_item.response.url, proxy_type=proxy_type,
-                                        project=self.project, task=self.task,
-                                    )
-                                parse_iterators = self.parse(download_item)
-                                if not isgenerator(parse_iterators):
-                                    raise TypeError("parse function isn't a generator")
-                                for parse_item in parse_iterators:
-                                    iterator_status = True
-                                    if isinstance(parse_item, Response):
-                                        raise TypeError("upload_item can't be a Response instance")
-                                    self.distribute(parse_item, seed)
-                            else:
-                                self.distribute(download_item, seed)
-                    else:
-                        self.distribute(request_item, seed)
-
-                if not iterator_status:
-                    raise ValueError("request/download/parse function yield value error!")
-            except Exception as e:
-                exception_msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                url = seed.url
-                status = e.__class__.__name__
-                if getattr(e, "response", None) and isinstance(e.response, Res):
-                    url = e.response.request.url
-                    status = e.response.status_code
-                self.loghub_dot.build(
-                    topic=urlparse(url).netloc,
-                    data_size=-1, cost_time=-1,
-                    status=status, url=url,
-                    proxy_type=proxy_type,
-                    project=self.project,
-                    task=self.task,
-                    msg=exception_msg,
-                )
-                logger.info(LogTemplate.download_exception.format(
-                    detail=seed_detail_log_info,
-                    retry=seed.params.retry,
-                    priority=seed.params.priority,
-                    seed_version=seed.params.seed_version,
-                    identifier=seed.identifier or "",
-                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                ))
-                seed.params.retry += 1
-                self._set_seed(seed)
-                # time.sleep(self.time_sleep * seed.params.retry)
-            # except Exception as e:
-            #     logger.info(LogTemplate.download_exception.format(
-            #         detail=seed_detail_log_info,
-            #         retry=seed.params.retry,
-            #         priority=seed.params.priority,
-            #         seed_version=seed.params.seed_version,
-            #         identifier=seed.identifier or "",
-            #         exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
-            #     ))
-            #     seed.params.retry += 1
-            #     # self._todo.push(seed)
-            #     self._set_seed(seed)
-            #     # time.sleep(self.time_sleep * seed.params.retry)
-            finally:
-                time.sleep(0.1)
-        logger.info("spider thread close")
+            else:
+                TaskQueue.process_task(seed, self.request)
+            TaskQueue.DELETE.push(member)
+        time.sleep(thread_sleep)
+
+    @Decorators.pause
+    def build_download_item(self):
+        thread_sleep = 0.1
+        if TaskQueue.RESPONSE.length >= self.download_queue_size:
+            logger.info(f"download queue is full, sleep {thread_sleep}s")
+            thread_sleep = 5
+        elif request_info := TaskQueue.DOWNLOAD.pop():
+            member, priority = request_info
+            request_setting = json.loads(member)
+            request_item = Request(seed=member, **request_setting)
+            TaskQueue.process_task(request_item, self.download)
+        time.sleep(thread_sleep)
+
+    @Decorators.pause
+    def build_parse_item(self):
+        thread_sleep = 0.1
+        if TaskQueue.UPLOAD.length >= self.upload_queue_size:
+            logger.info(f"upload queue is full, sleep {thread_sleep}s")
+            thread_sleep = 5
+        if response_item := TaskQueue.RESPONSE.pop():
+            TaskQueue.process_task(response_item, self.parse)
+        time.sleep(thread_sleep)
 
-    def run(self):
-        threading.Thread(name="loghub_dot", target=self.loghub_dot.build_run).start()
-        for index in range(self.thread_num):
-            threading.Thread(name=f"spider_{index}", target=self.spider).start()
 

cobweb/db/redis_db.py

@@ -27,6 +27,9 @@ class RedisDB:
     def exists(self, *name) -> bool:
         return self._client.exists(*name)
 
+    def incrby(self, name, value):
+        return self._client.incrby(name, value)
+
     def sadd(self, name, value):
         return self._client.sadd(name, value)
 
@@ -72,6 +75,31 @@ class RedisDB:
         status = self.execute_lua(lua_script, [key], t)
         return bool(status)
 
+    def auto_incr(self, name, t=15, limit=1000):
+        lua_script = """
+            local count = 0
+            local status = false
+            local limit = ARGV[2]
+            local expire = redis.call('ttl', KEYS[1])
+
+            if ( expire == -2 ) then
+                redis.call('setnx', KEYS[1], 1)
+            elseif ( expire == -1) then
+                redis.call('expire', KEYS[1], ARGV[1])
+            else
+                count = redis.call('get', KEYS[1])
+            end
+
+            if ( count + 0 < limit + 0 ) then
+                status = true
+                redis.call('incr', KEYS[1])
+            end
+
+            return status
+        """
+        status = self.execute_lua(lua_script, [name], t, limit)
+        return bool(status)
+
     def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
         lua_script = """
         local min = ARGV[1]

cobweb/launchers/__init__.py

@@ -1,3 +1,3 @@
-from .launcher_air import LauncherAir
+# from .launcher_air import LauncherAir
 from .launcher_pro import LauncherPro
-from .launcher_api import LauncherApi
+# from .launcher_api import LauncherApi

cobweb/launchers/launcher.py

@@ -2,63 +2,25 @@ import time
 import inspect
 import threading
 import importlib
-from functools import wraps
-
 
+from cobweb.constant import LogTemplate
+from cobweb.utils import dynamic_load_class
+from cobweb.base import TaskQueue, Decorators, logger
 from cobweb import setting
-from cobweb.base import Seed, Queue, logger
-from cobweb.utils.tools import dynamic_load_class
-
-
-def check_pause(func):
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
-        while not self._pause.is_set():
-            try:
-                func(self, *args, **kwargs)
-            except Exception as e:
-                logger.info(f"{func.__name__}: " + str(e))
-            finally:
-                time.sleep(0.1)
-
-    return wrapper
 
 
 class Launcher(threading.Thread):
 
-    SEEDS = []
-
-    __DOING__ = {}
-
-    __CUSTOM_FUNC__ = {
-        # "download": None,
-        # "request": None,
-        # "parse": None,
-    }
-
-    __LAUNCHER_QUEUE__ = {
-        "new": Queue(),
-        "todo": Queue(),
-        "done": Queue(),
-        "upload": Queue()
-    }
-
-    __LAUNCHER_FUNC__ = [
-        "_reset",
-        "_scheduler",
-        "_insert",
-        "_refresh",
-        "_delete",
-    ]
+    __CUSTOM_FUNC__ = {}
 
     def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__()
         self.task = task
         self.project = project
+        self.custom_func = dict()
+        self.app_time = int(time.time())
 
-        self._app_time = int(time.time())
-        self._stop = threading.Event()  # 结束事件
-        self._pause = threading.Event()  # 暂停事件
+        self.check_emtpy_times = 0
 
         _setting = dict()
 
@@ -79,32 +41,39 @@ class Launcher(threading.Thread):
         for k, v in _setting.items():
             setattr(setting, k.upper(), v)
 
-        self._Crawler = dynamic_load_class(setting.CRAWLER)
-        self._Pipeline = dynamic_load_class(setting.PIPELINE)
+        self.before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
+
+        self.scheduling_wait_time = setting.SCHEDULING_WAIT_TIME
+        self.inserting_wait_time = setting.INSERTING_WAIT_TIME
+        self.removing_wait_time = setting.REMOVING_WAIT_TIME
+        self.seed_reset_seconds = setting.SEED_RESET_SECONDS
+
+        self.scheduling_size = setting.SCHEDULING_SIZE
+        self.inserting_size = setting.INSERTING_SIZE
+        self.removing_size = setting.REMOVING_SIZE
 
-        self._before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
-        self._scheduler_wait_seconds = setting.SCHEDULER_WAIT_SECONDS
-        self._todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
-        self._new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
-        self._done_queue_wait_seconds = setting.DONE_QUEUE_WAIT_SECONDS
-        self._upload_queue_wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
-        self._seed_reset_seconds = setting.SEED_RESET_SECONDS
+        self.todo_queue_size = setting.TODO_QUEUE_SIZE
+        self.seed_queue_size = setting.SEED_QUEUE_SIZE
+        self.request_queue_size = setting.REQUEST_QUEUE_SIZE
+        self.download_queue_size = setting.DOWNLOAD_QUEUE_SIZE
+        self.response_queue_size = setting.RESPONSE_QUEUE_SIZE
+        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
+        self.delete_queue_size = setting.DELETE_QUEUE_SIZE
+        self.done_queue_size = setting.DONE_QUEUE_SIZE
 
-        self._todo_queue_size = setting.TODO_QUEUE_SIZE
-        self._new_queue_max_size = setting.NEW_QUEUE_MAX_SIZE
-        self._done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
-        self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
+        self.spider_thread_num = setting.SPIDER_THREAD_NUM
 
-        self._spider_max_retries = setting.SPIDER_MAX_RETRIES
-        self._spider_thread_num = setting.SPIDER_THREAD_NUM
-        self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
-        self._spider_max_count = setting.SPIDER_MAX_COUNT
-        self._time_window = setting.TIME_WINDOW
+        self.task_model = setting.TASK_MODEL
 
-        self._done_model = setting.DONE_MODEL
-        self._task_model = setting.TASK_MODEL
+        self.stop = threading.Event()  # 结束事件
+        self.pause = threading.Event()  # 暂停事件
 
-        self._filter_field = setting.FILTER_FIELD
+        self.crawler_path = setting.CRAWLER
+        self.pipeline_path = setting.PIPELINE
+
+        self._threads = []
+
+        self._task_info = dict(todo={}, download={})
 
     @property
     def request(self):
@@ -118,7 +87,7 @@ class Launcher(threading.Thread):
                 yield Request(seed.url, seed)
         """
         def decorator(func):
-            self.__CUSTOM_FUNC__["request"] = func
+            self.custom_func['request'] = func
         return decorator
 
     @property
@@ -133,7 +102,7 @@ class Launcher(threading.Thread):
                 yield Response(item.seed, response)
         """
         def decorator(func):
-            self.__CUSTOM_FUNC__["download"] = func
+            self.custom_func['download'] = func
         return decorator
 
     @property
@@ -148,88 +117,88 @@ class Launcher(threading.Thread):
                yield xxxItem(seed, **kwargs)
         """
         def decorator(func):
-            self.__CUSTOM_FUNC__["parse"] = func
+            self.custom_func['parse'] = func
         return decorator
 
-    def start_seeds(self):
-        seeds = [Seed(seed) for seed in self.SEEDS]
-        self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-        return seeds
+    def remove_working_items(self, key, items):
+        for item in items:
+            self._task_info[key].pop(item, None)
 
-    def _remove_doing_seeds(self, seeds):
-        for seed in seeds:
-            self.__DOING__.pop(seed, None)
-        # logger.info("remove %s seeds from __DOING__" % len(seeds))
+    def add_working_item(self, key, member, priority):
+        self._task_info[key][member] = priority
 
-    def _get_seed(self) -> Seed:
-        return self.__LAUNCHER_QUEUE__["todo"].pop()
+    def check_alive(self):
+        while not self.stop.is_set():
+            if not self.pause.is_set():
+                for thread in self._threads:
+                    if not thread.is_alive():
+                        thread.start()
+            time.sleep(1)
 
-    def _set_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["todo"].push(seed, **kwargs)
+    def _add_thread(self, func, num=1, obj=None, name=None, args=()):
+        obj = obj or self
+        name = obj.__class__.__name__ + name or func.__name__
+        for i in range(num):
+            func_name = name + "_" + str(i) if num > 1 else name
+            self._threads.append(threading.Thread(name=func_name, target=func, args=(obj,) + args))
+
+    @Decorators.stop
+    def _polling(self):
+        if TaskQueue.is_empty():
+            if self.pause.is_set():
+                run_time = int(time.time()) - self.app_time
+                if not self.task_model and run_time > self.before_scheduler_wait_seconds:
+                    logger.info("Done! ready to close thread...")
+                    self.stop.set()
+                else:
+                    logger.info("pause! waiting for resume...")
+            elif self.check_emtpy_times > 2:
+                logger.info("pause! waiting for resume...")
+                self.doing_seeds = {}
+                self._task_info['todo'] = {}
+                self._task_info['download'] = {}
+                self.pause.set()
+            else:
+                logger.info(
+                    "check whether the task is complete, "
+                    f"reset times {3 - self.check_emtpy_times}"
+                )
+                self.check_emtpy_times += 1
+        elif TaskQueue.TODO.length:
+            logger.info(f"Recovery {self.task} task run！")
+            self.check_emtpy_times = 0
+            self.pause.clear()
+        else:
+            logger.info(LogTemplate.launcher_polling.format(
+                task=self.task,
+                memory_todo_count=len(self._task_info["todo"]),
+                memory_download_count=len(self._task_info["download"]),
+                todo_queue_len=TaskQueue.TODO.length,
+                delete_queue_len=TaskQueue.DELETE.length,
+                request_queue_len=TaskQueue.REQUEST.length,
+                response_queue_len=TaskQueue.RESPONSE.length,
+                done_queue_len=TaskQueue.DONE.length,
+                upload_queue_len=TaskQueue.UPLOAD.length,
+            ))
+        time.sleep(10)
 
-    def _upload_data(self, data, **kwargs):
-        self.__LAUNCHER_QUEUE__["upload"].push(data, **kwargs)
+    def run(self):
+        Crawler = dynamic_load_class(self.crawler_path)
+        Pipeline = dynamic_load_class(self.pipeline_path)
 
-    def _add_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["new"].push(seed, **kwargs)
+        crawler = Crawler(pause=self.pause, custom_func=self.custom_func)
+        pipeline = Pipeline(pause=self.pause)
 
-    def _delete_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["done"].push(seed, **kwargs)
+        self._add_thread(obj=crawler, func=crawler.build_request_item)
+        self._add_thread(obj=crawler, func=crawler.build_download_item, num=self.spider_thread_num)
+        self._add_thread(obj=crawler, func=crawler.build_parse_item)
+        self._add_thread(obj=pipeline, func=pipeline.run)
 
-    def _execute(self):
-        for func_name in self.__LAUNCHER_FUNC__:
-            threading.Thread(name=func_name, target=getattr(self, func_name)).start()
-            time.sleep(1)
+        self._add_thread(func=self._polling)
 
-    def run(self):
-        threading.Thread(target=self._execute_heartbeat).start()
-
-        self.start_seeds()
-
-        self._Crawler(
-            task=self.task, project=self.project,
-            stop=self._stop, pause=self._pause,
-            # launcher_queue=self.__LAUNCHER_QUEUE__,
-            get_seed=self._get_seed,
-            set_seed=self._set_seed,
-            add_seed=self._add_seed,
-            delete_seed=self._delete_seed,
-            upload_data=self._upload_data,
-            custom_func=self.__CUSTOM_FUNC__,
-            thread_num = self._spider_thread_num,
-            max_retries = self._spider_max_retries,
-            time_sleep=self._spider_time_sleep
-        ).start()
-
-        self._Pipeline(
-            stop=self._stop, pause=self._pause,
-            upload=self.__LAUNCHER_QUEUE__["upload"],
-            done=self.__LAUNCHER_QUEUE__["done"],
-            upload_size=self._upload_queue_max_size,
-            wait_seconds=self._upload_queue_wait_seconds
-        ).start()
-
-        self._execute()
-        self._polling()
-
-    def _execute_heartbeat(self):
-        pass
-
-    def _reset(self):
-        pass
-
-    def _scheduler(self):
-        pass
-
-    def _insert(self):
-        pass
-
-    def _refresh(self):
-        pass
-
-    def _delete(self):
-        pass
+        self._init_schedule_thread()
+        self.check_alive()
 
-    def _polling(self):
-        pass
+    def _init_schedule_thread(self):
+        ...