cobweb-launcher 1.2.25__py3-none-any.whl → 3.2.18__py3-none-any.whl

cobweb/db/redis_db.py

@@ -1,64 +1,156 @@
+import os
+
+import time
 import redis
-from cobweb import setting
+from redis.exceptions import ConnectionError, TimeoutError
 
 
 class RedisDB:
-
-    def __init__(self, **kwargs):
-        redis_config = kwargs or setting.REDIS_CONFIG
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
+    def __init__(
+            self,
+            host=None,
+            password=None,
+            port=6379, db=0
+    ):
+        self.host = host or os.getenv("REDIS_HOST", "localhost")
+        self.password = password or os.getenv("REDIS_PASSWORD")
+        self.port = port or os.getenv("REDIS_PORT", 6379)
+        self.db = db or os.getenv("REDIS_DB", 0)
+
+        self.max_retries = 5
+        self.retry_delay = 5
+        self.client = None
+        self.connect()
+
+    def connect(self):
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                self.client = redis.Redis(
+                    host=self.host,
+                    port=self.port,
+                    password=self.password,
+                    db=self.db,
+                    socket_timeout=5,
+                    socket_connect_timeout=5
+                )
+                self.client.ping()
+                return
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法连接 Redis")
+
+    def is_connected(self):
+        try:
+            self.client.ping()
+            return True
+        except (ConnectionError, TimeoutError):
+            return False
+
+    def reconnect(self):
+        self.connect()
+
+    def execute_command(self, command, *args, **kwargs):
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                if not self.is_connected():
+                    self.reconnect()
+                return getattr(self.client, command)(*args, **kwargs)
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法执行命令")
+
+    def get(self, name):
+        # with self.get_connection() as client:
+        #     return client.get(name)
+        return self.execute_command("get", name)
+
+    def incrby(self, name, value):
+        # with self.get_connection() as client:
+        #     client.incrby(name, value)
+        self.execute_command("incrby", name, value)
 
     def setnx(self, name, value=""):
-        return self._client.setnx(name, value)
+        # with self.get_connection() as client:
+        #     client.setnx(name, value)
+        self.execute_command("setnx", name, value)
 
     def setex(self, name, t, value=""):
-        return self._client.setex(name, t, value)
+        # with self.get_connection() as client:
+        #     client.setex(name, t, value)
+        self.execute_command("setex", name, t, value)
 
     def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        return self._client.expire(name, t, nx, xx, gt, lt)
+        # with self.get_connection() as client:
+        # client.expire(name, t, nx, xx, gt, lt)
+        self.execute_command("expire", name, t, nx, xx, gt, lt)
 
     def ttl(self, name):
-        return self._client.ttl(name)
+        # with self.get_connection() as client:
+        #     return client.ttl(name)
+        return self.execute_command("ttl", name)
 
     def delete(self, name):
-        return self._client.delete(name)
+        # with self.get_connection() as client:
+        #     return client.delete(name)
+        return self.execute_command("delete", name)
 
     def exists(self, *name) -> bool:
-        return self._client.exists(*name)
+        # with self.get_connection() as client:
+        #     return client.exists(*name)
+        return self.execute_command("exists", *name)
 
     def sadd(self, name, value):
-        return self._client.sadd(name, value)
+        # with self.get_connection() as client:
+        #     return client.sadd(name, value)
+        return self.execute_command("sadd", name, value)
 
     def zcard(self, name) -> bool:
-        return self._client.zcard(name)
+        # with self.get_connection() as client:
+        #     return client.zcard(name)
+        return self.execute_command("zcard", name)
 
     def zadd(self, name, item: dict, **kwargs):
-        return self._client.zadd(name, item, **kwargs)
+        # with self.get_connection() as client:
+        #     return client.zadd(name, item, **kwargs)
+        if item:
+            return self.execute_command("zadd", name, item, **kwargs)
 
     def zrem(self, name, *value):
-        return self._client.zrem(name, *value)
+        # with self.get_connection() as client:
+        #     return client.zrem(name, *value)
+        return self.execute_command("zrem", name, *value)
 
     def zcount(self, name, _min, _max):
-        return self._client.zcount(name, _min, _max)
+        # with self.get_connection() as client:
+        #     return client.zcount(name, _min, _max)
+        return self.execute_command("zcount", name, _min, _max)
 
     # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
-    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+    #     with self.get_connection() as client:
+    #        return client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
 
     def lua(self, script: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        return self._client.eval(script, keys_count, *keys, *args)
+        return self.execute_command("eval", script, keys_count, *keys, *args)
 
     def lua_sha(self, sha1: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        return self._client.evalsha(sha1, keys_count, *keys, *args)
+        return self.execute_command("evalsha", sha1, keys_count, *keys, *args)
 
     def execute_lua(self, lua_script: str, keys: list, *args):
-        execute = self._client.register_script(lua_script)
+        execute = self.execute_command("register_script", lua_script)
         return execute(keys=keys, args=args)
 
     def lock(self, key, t=15) -> bool:
@@ -72,7 +164,7 @@ class RedisDB:
         status = self.execute_lua(lua_script, [key], t)
         return bool(status)
 
-    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
+    def members(self, key, score, start=0, count=1000, _min="-inf", _max="+inf") -> list:
         lua_script = """
         local min = ARGV[1]
         local max = ARGV[2]
@@ -86,7 +178,7 @@ class RedisDB:
         else
             members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
         end
-        
+
         local result = {}
 
         for i = 1, #members, 2 do
@@ -98,7 +190,7 @@ class RedisDB:
             else
                 originPriority = math.floor(members[i+1])
             end
-            
+
             if ( score + 0 >= 1000 ) then
                 priority = -score - originPriority / 1000
             elseif ( score + 0 == 0 ) then
@@ -117,7 +209,7 @@ class RedisDB:
         members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
         return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
 
-    def done(self, keys: list, *args) -> list:
+    def done(self, keys: list, *args):
         lua_script = """
         for i, member in ipairs(ARGV) do
             redis.call("zrem", KEYS[1], member)
@@ -125,6 +217,3 @@ class RedisDB:
         end
         """
         self.execute_lua(lua_script, keys, *args)
-
-
-

cobweb/launchers/__init__.py

@@ -1,3 +1,4 @@
-from .launcher_air import LauncherAir
-from .launcher_pro import LauncherPro
-from .launcher_api import LauncherApi
+from .launcher import Launcher
+from .uploader import Uploader
+from .distributor import Distributor
+

cobweb/launchers/distributor.py

@@ -0,0 +1,141 @@
+import time
+import threading
+import traceback
+
+from inspect import isgenerator
+from typing import Callable, Type
+from requests import RequestException
+
+from cobweb.crawlers import Crawler
+from cobweb.utils import check_pause
+from cobweb.log_dots import LoghubDot
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.base import Seed, Status, TaskQueue, BaseItem, Request, Response, logger
+
+
+class Distributor(threading.Thread):
+
+    def __init__(
+            self,
+            task: str,
+            project: str,
+            task_queue: TaskQueue,
+            stop: threading.Event,
+            pause: threading.Event,
+            callback_register: Callable,
+            SpiderCrawler: Type[Crawler]
+    ):
+        super().__init__()
+        self.task = task
+        self.project = project
+        self.pause = pause
+
+        self.task_queue = task_queue
+
+        self.callback_register = callback_register
+        self.Crawler = SpiderCrawler
+
+        from cobweb import setting
+        self.time_sleep = setting.SPIDER_TIME_SLEEP
+        self.thread_num = setting.SPIDER_THREAD_NUM
+        self.max_retries = setting.SPIDER_MAX_RETRIES
+        self.loghub_dot = LoghubDot(stop=stop, project=self.project, task=self.task)
+
+        logger.debug(f"Distribute instance attrs: {self.__dict__}")
+
+    def distribute(self, task_id, item, status: Status):
+        if isinstance(item, Request):
+            item.seed.params.request_time = time.time()
+            self.loghub_dot._build_request_log(item)
+            self.process(task_id=task_id, item=item, callback=self.Crawler.download, status=Status.PROCESSING)
+
+        elif isinstance(item, Response):
+            if status == Status.FINISHED:
+                raise TypeError("parse function can't yield a Response instance")
+            item.seed.params.download_time = time.time()
+            logger.debug(LogTemplate.download_info.format(
+                detail=LogTemplate.log_info(item.seed.to_dict),
+                retry=item.seed.params.retry,
+                priority=item.seed.params.priority,
+                seed_version=item.seed.params.seed_version,
+                identifier=item.seed.identifier or "",
+                status=item.response,
+                response=LogTemplate.log_info(item.to_dict)
+            ))
+            self.loghub_dot._build_download_log(item)
+            self.process(task_id=task_id, item=item, callback=self.Crawler.parse, status=Status.FINISHED)
+
+        elif isinstance(item, BaseItem):
+            item.seed.params.parse_time = time.time()
+            self.loghub_dot._build_parse_log(item)
+            self.task_queue.add_task(data=item, status=Status.UPLOAD, parent_id=task_id)
+
+        elif isinstance(item, Seed):
+            # todo: 新种子日志
+            item.seed.params.insert_time = time.time()
+            self.task_queue.add_task(
+                task_id=item.sid, data=item, status=Status.INSERT,
+                priority=item.params.priority, parent_id=task_id
+            )
+
+        elif isinstance(item, str) and item != DealModel.done:
+            raise TypeError("yield value type error!")
+
+    def process(self, task_id, item, callback, status: Status):
+        iterators = callback(item)
+        if not isgenerator(iterators):
+            raise TypeError(f"{callback.__name__} function isn't a generator!")
+        for it in iterators:
+            self.distribute(task_id=task_id, item=it, status=status)
+
+    @check_pause
+    def spider(self):
+        if task_item := self.task_queue.get_pending_task():
+            finsh_status = True
+            seed = task_item.data
+            status = Status.FINISHED
+            task_id = task_item.task_id
+            seed.params.start_time = time.time()
+
+            if seed.params.retry and isinstance(seed.params.retry, int):
+                time.sleep(self.time_sleep * seed.params.retry / 100)
+
+            try:
+                self.process(task_id=task_id, item=seed, callback=self.Crawler.request, status=Status.PENDING)
+            except Exception as e:
+                seed.params.retry += 1
+                seed.params.failed_time = time.time()
+                msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                if not seed.params.msg:
+                    seed.params.traceback = [msg]
+                elif isinstance(seed.params.msg, list):
+                    seed.params.traceback.append(msg)
+
+                if isinstance(e, RequestException):
+                    self.loghub_dot._build_http_error_log(seed, e)
+                else:
+                    self.loghub_dot._build_exception_log(seed, e)
+
+                if seed.params.retry < self.max_retries:
+                    status = Status.PENDING
+                    finsh_status = False
+
+                logger.info(LogTemplate.download_exception.format(
+                    detail=LogTemplate.log_info(seed.to_dict),
+                    retry=seed.params.retry,
+                    priority=seed.params.priority,
+                    seed_version=seed.params.seed_version,
+                    identifier=seed.identifier or "",
+                    exception=msg
+                ))
+
+            finally:
+                if finsh_status:
+                    seed.params.finsh_time = time.time()
+                    self.loghub_dot._build_finish_log(seed, status=bool(seed.params.retry < self.max_retries))
+                self.task_queue.update_task(task_id, status=status, data=seed)
+
+    def run(self):
+        self.callback_register(self.loghub_dot._build_run, tag="LoghubDot")
+        for _ in range(self.thread_num):
+            self.callback_register(self.spider, tag="Distributor")

cobweb/launchers/launcher.py

@@ -1,57 +1,25 @@
 import time
+import uuid
 import inspect
 import threading
 import importlib
-from functools import wraps
 
 from cobweb import setting
-from cobweb.base import Seed, Queue, logger
+from cobweb.launchers.uploader import Uploader
 from cobweb.utils.tools import dynamic_load_class
+from cobweb.launchers.distributor import Distributor
+from cobweb.base import Seed, logger, TaskQueue, Status
+from typing import Optional, Union, Dict, Any, Callable
 
 
-def check_pause(func):
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
-        while not self._pause.is_set():
-            try:
-                func(self, *args, **kwargs)
-            except Exception as e:
-                logger.info(f"{func.__name__}: " + str(e))
-            finally:
-                time.sleep(0.1)
+class Launcher:
 
-    return wrapper
-
-
-class Launcher(threading.Thread):
-
-    SEEDS = []
-
-    __DOING__ = {}
-
-    __CUSTOM_FUNC__ = {
-        # "download": None,
-        # "request": None,
-        # "parse": None,
-    }
-
-    __LAUNCHER_QUEUE__ = {
-        "new": Queue(),
-        "todo": Queue(),
-        "done": Queue(),
-        "upload": Queue()
-    }
-
-    __LAUNCHER_FUNC__ = [
-        "_reset",
-        "_scheduler",
-        "_insert",
-        "_refresh",
-        "_delete",
-    ]
+    __REGISTER_FUNC__: Dict[str, Callable] = {}
+    __WORKER_THREAD__: Dict[str, threading.Thread] = {}
 
     def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__()
+
         self.task = task
         self.project = project
 
@@ -59,51 +27,37 @@ class Launcher(threading.Thread):
         self._stop = threading.Event()  # 结束事件
         self._pause = threading.Event()  # 暂停事件
 
-        _setting = dict()
+        _setting = self._load_custom_settings(custom_setting)
 
-        if custom_setting:
-            if isinstance(custom_setting, dict):
-                _setting = custom_setting
-            else:
-                if isinstance(custom_setting, str):
-                    custom_setting = importlib.import_module(custom_setting)
-                if not inspect.ismodule(custom_setting):
-                    raise Exception
-                for k, v in custom_setting.__dict__.items():
-                    if not k.startswith("__") and not inspect.ismodule(v):
-                        _setting[k] = v
-
-        _setting.update(**kwargs)
-
-        for k, v in _setting.items():
-            setattr(setting, k.upper(), v)
-
-        self._Crawler = dynamic_load_class(setting.CRAWLER)
-        self._Pipeline = dynamic_load_class(setting.PIPELINE)
-
-        self._before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
-        self._scheduler_wait_seconds = setting.SCHEDULER_WAIT_SECONDS
-        self._todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
-        self._new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
-        self._done_queue_wait_seconds = setting.DONE_QUEUE_WAIT_SECONDS
-        self._upload_queue_wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
-        self._seed_reset_seconds = setting.SEED_RESET_SECONDS
-
-        self._todo_queue_size = setting.TODO_QUEUE_SIZE
-        self._new_queue_max_size = setting.NEW_QUEUE_MAX_SIZE
-        self._done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
-        self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
-
-        self._spider_max_retries = setting.SPIDER_MAX_RETRIES
-        self._spider_thread_num = setting.SPIDER_THREAD_NUM
-        self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
-        self._spider_max_count = setting.SPIDER_MAX_COUNT
-        self._time_window = setting.TIME_WINDOW
+        _setting.update(kwargs)
+        for key, value in _setting.items():
+            setattr(setting, key.upper(), value)
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL
 
-        self._filter_field = setting.FILTER_FIELD
+        self._task_queue = TaskQueue()
+
+        self.Scheduler = dynamic_load_class(setting.SCHEDULER)
+        self.SpiderCrawler = dynamic_load_class(setting.CRAWLER)
+        self.SpiderPipeline = dynamic_load_class(setting.PIPELINE)
+
+    @staticmethod
+    def _load_custom_settings(custom_setting: Optional[Union[str, Dict]]) -> Dict[str, Any]:
+        _setting = {}
+        if custom_setting:
+            if isinstance(custom_setting, dict):
+                _setting = custom_setting
+            elif isinstance(custom_setting, str):
+                module = importlib.import_module(custom_setting)
+                _setting = {
+                    k: v
+                    for k, v in module.__dict__.items()
+                    if not k.startswith("__") and not inspect.ismodule(v)
+                }
+            else:
+                raise ValueError("custom_setting must be a dictionary or a module path.")
+        return _setting
 
     @property
     def request(self):
@@ -117,7 +71,7 @@ class Launcher(threading.Thread):
                 yield Request(seed.url, seed)
         """
         def decorator(func):
-            self.__CUSTOM_FUNC__["request"] = func
+            self.SpiderCrawler.request = func
         return decorator
 
     @property
@@ -132,7 +86,7 @@ class Launcher(threading.Thread):
                 yield Response(item.seed, response)
         """
         def decorator(func):
-            self.__CUSTOM_FUNC__["download"] = func
+            self.SpiderCrawler.download = func
         return decorator
 
     @property
@@ -147,87 +101,71 @@ class Launcher(threading.Thread):
                yield xxxItem(seed, **kwargs)
         """
         def decorator(func):
-            self.__CUSTOM_FUNC__["parse"] = func
+            self.SpiderCrawler.parse = func
         return decorator
 
-    def start_seeds(self):
-        seeds = [Seed(seed) for seed in self.SEEDS]
-        self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-        return seeds
-
-    def _remove_doing_seeds(self, seeds):
-        for seed in seeds:
-            self.__DOING__.pop(seed, None)
-        # logger.info("remove %s seeds from __DOING__" % len(seeds))
-
-    def _get_seed(self) -> Seed:
-        return self.__LAUNCHER_QUEUE__["todo"].pop()
-
-    def _set_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["todo"].push(seed, **kwargs)
-
-    def _upload_data(self, data, **kwargs):
-        self.__LAUNCHER_QUEUE__["upload"].push(data, **kwargs)
-
-    def _add_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["new"].push(seed, **kwargs)
-
-    def _delete_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["done"].push(seed, **kwargs)
-
-    def _execute(self):
-        for func_name in self.__LAUNCHER_FUNC__:
-            threading.Thread(name=func_name, target=getattr(self, func_name)).start()
-            time.sleep(1)
-
-    def run(self):
-        threading.Thread(target=self._execute_heartbeat).start()
-
-        self.start_seeds()
+    def start_seeds(self, seeds: list[Union[str, Dict]]) -> list[Seed]:
+        seed_list = [Seed(seed) for seed in seeds]
+        for seed in seed_list:
+            self._task_queue.add_task(
+                task_id=seed.sid,
+                data=seed,
+                status=Status.PENDING,
+                priority=seed.params.priority,
+                parent_id=None,
+                ttl_seconds=None
+            )
+        return seed_list
+
+    def _register(self, func: Callable, tag: str = "launcher"):
+        name = f"{tag}:{func.__name__}_{uuid.uuid4()}"
+        self.__REGISTER_FUNC__[name] = func
+        if not self.__WORKER_THREAD__.get(name):
+            worker_thread = threading.Thread(name=name, target=func)
+            self.__WORKER_THREAD__[name] = worker_thread
+
+    def _monitor(self):
+        while not self._stop.is_set():
+            if not self._pause.is_set():
+                for name, worker_thread in list(self.__WORKER_THREAD__.items()):
+                    if not worker_thread.is_alive():
+                        logger.debug(f"{name} thread is dead. Restarting...")
+                        func = self.__REGISTER_FUNC__[name]
+                        worker_thread = threading.Thread(name=name, target=func)
+                        self.__WORKER_THREAD__[name] = worker_thread
+                        worker_thread.start()
+            time.sleep(15)
+        logger.info("monitor thread close!")
+
+    def start(self):
+        self._pause.is_set()
+
+        self.Scheduler(
+            task=self.task,
+            project=self.project,
+            stop=self._stop,
+            pause=self._pause,
+            task_queue=self._task_queue,
+            callback_register=self._register
+        ).start()
 
-        self._Crawler(
+        Distributor(
+            task=self.task,
+            project=self.project,
+            task_queue=self._task_queue,
+            callback_register=self._register,
             stop=self._stop, pause=self._pause,
-            # launcher_queue=self.__LAUNCHER_QUEUE__,
-            get_seed=self._get_seed,
-            set_seed=self._set_seed,
-            add_seed=self._add_seed,
-            delete_seed=self._delete_seed,
-            upload_data=self._upload_data,
-            custom_func=self.__CUSTOM_FUNC__,
-            thread_num = self._spider_thread_num,
-            max_retries = self._spider_max_retries,
-            time_sleep=self._spider_time_sleep
+            SpiderCrawler=self.SpiderCrawler
         ).start()
 
-        self._Pipeline(
+        Uploader(
+            task=self.task, project=self.project,
             stop=self._stop, pause=self._pause,
-            upload=self.__LAUNCHER_QUEUE__["upload"],
-            done=self.__LAUNCHER_QUEUE__["done"],
-            upload_size=self._upload_queue_max_size,
-            wait_seconds=self._upload_queue_wait_seconds
+            task_queue=self._task_queue,
+            callback_register=self._register,
+            SpiderPipeline=self.SpiderPipeline
         ).start()
 
-        self._execute()
-        self._polling()
-
-    def _execute_heartbeat(self):
-        pass
-
-    def _reset(self):
-        pass
-
-    def _scheduler(self):
-        pass
-
-    def _insert(self):
-        pass
-
-    def _refresh(self):
-        pass
-
-    def _delete(self):
-        pass
-
-    def _polling(self):
-        pass
+        self._monitor()
+        logger.info("task done!")