cobweb-launcher 1.2.25__py3-none-any.whl → 3.2.20__py3-none-any.whl

cobweb/schedulers/scheduler_with_redis.py

@@ -0,0 +1,189 @@
+import os
+import time
+import threading
+from typing import Callable
+from cobweb.db import RedisDB, ApiDB
+from cobweb.utils import check_pause
+from cobweb.base import Seed, logger, TaskQueue, Status
+from cobweb.constant import LogTemplate
+from .scheduler import Scheduler
+use_api = bool(os.getenv("REDIS_API_HOST", 0))
+
+
+class RedisScheduler(Scheduler):
+
+    def __init__(
+            self,
+            task,
+            project,
+            stop: threading.Event,
+            pause: threading.Event,
+            task_queue: TaskQueue,
+            callback_register: Callable
+    ):
+        super().__init__(task, project, stop, pause, task_queue, callback_register)
+        self.todo_key = f"{{{project}:{task}}}:todo"
+        self.done_key = f"{{{project}:{task}}}:done"
+        self.fail_key = f"{{{project}:{task}}}:fail"
+        self.heartbeat_key = f"heartbeat:{project}_{task}"
+        self.heartbeat_run_key = f"run:{project}_{task}"
+        self.speed_control_key = f"speed_control:{project}_{task}"
+        self.reset_lock_key = f"lock:reset:{project}_{task}"
+        self.db = ApiDB() if use_api else RedisDB()
+
+    def reset(self):
+        """
+        检查过期种子，重新添加到redis缓存中
+        """
+        while not self.stop.is_set():
+            if self.db.lock(self.reset_lock_key, t=360):
+
+                _min = -int(time.time()) + self.seed_reset_seconds
+                self.db.members(self.todo_key, 0, _min=_min, _max="(0")
+                self.db.delete(self.reset_lock_key)
+
+            time.sleep(self.seed_reset_seconds)
+
+    @check_pause
+    def schedule(self):
+        """
+        调度任务，获取redis队列种子，同时添加到doing字典中
+        """
+        if not self.db.zcount(self.todo_key, 0, "(1000"):
+            time.sleep(self.scheduler_wait_seconds)
+            return
+
+        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size\
+                or self.task_queue.length() > 5 * self.todo_queue_size:
+            time.sleep(self.todo_queue_full_wait_seconds)
+            return
+
+        if members := self.db.members(
+            self.todo_key, int(time.time()),
+            count=self.todo_queue_size,
+            _min=0, _max="(1000"
+        ):
+            for member, priority in members:
+                seed = Seed(member, priority=int(priority % 1000))
+                seed.params.get_time = time.time()
+                self.task_queue.add_task(
+                    task_id=seed.sid, data=seed,
+                    status=Status.PENDING,
+                    priority=seed.params.priority
+                )
+
+    @check_pause
+    def insert(self):
+        """
+        添加新种子到redis队列中
+        """
+        if task_list := self.task_queue.get_task_by_status(
+            status=Status.INSERT, limit=self.new_queue_max_size
+        ):
+            seed_info, task_ids = dict(), set()
+
+            for task_item in task_list:
+                seed = task_item.data
+                task_ids.add(task_item.task_id)
+                seed_info[seed.to_string] = seed.params.priority
+
+            self.db.zadd(self.todo_key, seed_info, nx=True)
+            self.task_queue.remove(task_ids)
+
+        if self.task_queue.status_length(status=Status.INSERT) < self.new_queue_max_size:
+            time.sleep(self.scheduler_wait_seconds)
+
+    @check_pause
+    def refresh(self):
+        """
+        刷新doing种子过期时间，防止reset重新消费
+        """
+        if task_list := self.task_queue.get_task_by_status(
+            status=[Status.PENDING, Status.PROCESSING, Status.FINISHED],
+        ):
+            refresh_time = int(time.time())
+            seed_info = {it.data.to_string: -refresh_time - it.data.params.priority / 1000 for it in task_list}
+            self.db.zadd(self.todo_key, seed_info, xx=True)
+        time.sleep(self.seed_reset_seconds // 3)
+
+    @check_pause
+    def delete(self):
+        """
+        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
+        """
+        if task_list := self.task_queue.get_task_by_status(
+                status=Status.FINISHED, limit=self.done_queue_max_size
+        ):
+            zrem_items = [it.data.to_string for it in task_list]
+            remove_task_ids = [it.task_id for it in task_list]
+            self.db.zrem(self.todo_key, *zrem_items)
+            self.task_queue.remove(remove_task_ids)
+
+        if self.task_queue.status_length(status=Status.FINISHED) < self.done_queue_max_size:
+            time.sleep(self.done_queue_wait_seconds)
+
+    def run(self):
+        start_time = int(time.time())
+
+        for func in [self.reset, self.insert, self.delete, self.refresh, self.schedule]:
+            self.callback_register(func, tag="scheduler")
+
+        while not self.stop.is_set():
+            todo_len = self.task_queue.status_length(status=Status.PENDING)
+            doing_len = self.task_queue.status_length(status=Status.PROCESSING)
+            done_len = self.task_queue.status_length(status=Status.FINISHED)
+            upload_len = self.task_queue.status_length(status=Status.UPLOAD)
+
+            redis_doing_count = self.db.zcount(self.todo_key, "-inf", "(0")
+            redis_todo_len = self.db.zcount(self.todo_key, 0, "(1000")
+            redis_seed_count = self.db.zcard(self.todo_key)
+
+            if self.pause.is_set():
+                execute_time = int(time.time()) - start_time
+                if redis_todo_len and self.task_queue.length() > 0:
+                    logger.info(
+                        f"Recovery {self.task} task run！"
+                        f"Todo seeds count: {redis_todo_len}"
+                        f", queue length: {redis_seed_count}"
+                    )
+                    self.pause.clear()
+                elif not self.task_model and execute_time > self.before_scheduler_wait_seconds:
+                    logger.info("Done! ready to close thread...")
+                    self.stop.set()
+                else:
+                    logger.info("Pause! waiting for resume...")
+
+            elif self.task_queue.length() == 0:
+                if redis_seed_count:
+                    logger.info(
+                        f"Todo seeds count: {redis_todo_len}"
+                        f", queue length: {redis_seed_count}"
+                    )
+                    self.pause.clear()
+                else:
+                    count = 0
+                    for _ in range(3):
+                        if not redis_seed_count:
+                            count += 1
+                            time.sleep(5)
+                            logger.info("Checking count...")
+                        else:
+                            break
+                    if count >= 3:
+                        logger.info("Todo queue is empty! Pause set...")
+                        self.pause.set()
+
+            else:
+                self.db.setex(self.heartbeat_run_key, 60, 1)
+                logger.info(LogTemplate.launcher_pro_polling.format(
+                    task=self.task,
+                    doing_len=doing_len,
+                    todo_len=todo_len,
+                    done_len=done_len,
+                    redis_seed_count=redis_seed_count,
+                    redis_todo_len=redis_todo_len,
+                    redis_doing_len=redis_doing_count,
+                    upload_len=upload_len,
+                ))
+
+            time.sleep(30)

cobweb/setting.py

@@ -1,37 +1,11 @@
-import os
-
-# redis db config
-REDIS_CONFIG = {
-    "host": os.getenv("REDIS_HOST"),
-    "password": os.getenv("REDIS_PASSWORD"),
-    "port": int(os.getenv("REDIS_PORT", 6379)),
-    "db": int(os.getenv("REDIS_DB", 0)),
-}
-
-# loghub db config
-LOGHUB_TOPIC = os.getenv("LOGHUB_TOPIC")
-LOGHUB_SOURCE = os.getenv("LOGHUB_SOURCE")
-LOGHUB_PROJECT = os.getenv("LOGHUB_PROJECT")
-LOGHUB_CONFIG = {
-    "endpoint": os.getenv("LOGHUB_ENDPOINT"),
-    "accessKeyId": os.getenv("LOGHUB_ACCESS_KEY"),
-    "accessKey": os.getenv("LOGHUB_SECRET_KEY")
-}
-
-# oss util config
-OSS_BUCKET = os.getenv("OSS_BUCKET")
-OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
-OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
-OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
-OSS_CHUNK_SIZE = 10 * 1024 ** 2
-OSS_MIN_UPLOAD_SIZE = 1024
-
-
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
 
-# 数据存储链路
-PIPELINE = "cobweb.pipelines.pipeline_console.Console"
+# 数据管道
+PIPELINE = "cobweb.pipelines.CSV"
+
+# 调度器
+SCHEDULER = "cobweb.schedulers.RedisScheduler"
 
 
 # Launcher 等待时间
@@ -42,7 +16,7 @@ TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
 DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
 UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
-SEED_RESET_SECONDS = 30   # 种子重制时间
+SEED_RESET_SECONDS = 60   # 种子重制时间
 
 
 # Launcher 队列长度
@@ -58,17 +32,30 @@ DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
 SPIDER_TIME_SLEEP = 10
+RECORD_FAILED_SPIDER = True
 
 SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
 TIME_WINDOW = 60   # 频控固定时间窗口（秒）
 
-# 任务模式
-TASK_MODEL = 0  # 0:单次，1:常驻
+# 任务模式, 0:单次，1:常驻
+TASK_MODEL = 0
+
+# 流控措施, 0:关闭，1:开启
+SPEED_CONTROL = 1
 
+DOT = 0
+
+# redis config
+REDIS_CONFIG = {
+    "host": "127.0.0.1",
+    "port": 6379,
+    "db": 0
+}
 
-# bloom过滤器
-CAPACITY = 100000000
-ERROR_RATE = 0.001
-FILTER_FIELD = "url"
-# 文件下载响应类型过滤
-# FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]
+# loghub pipeline config
+# os.getenv("LOGHUB_ENDPOINT"),
+# os.getenv("LOGHUB_ACCESS_KEY"),
+# os.getenv("LOGHUB_SECRET_KEY")
+# os.getenv("LOGHUB_PROJECT")
+# os.getenv("LOGHUB_SOURCE")
+# os.getenv("LOGHUB_TOPIC")

cobweb/utils/__init__.py

@@ -1,4 +1,6 @@
-from .oss import OssUtil
-from .tools import *
-from .bloom import BloomFilter
+# from .oss import OssUtil
+# from .bloom import BloomFilter
+# from .dotting import LoghubDot
+from .decorators import check_pause
+from .tools import md5, dynamic_load_class
 

cobweb/utils/bloom.py

@@ -1,58 +1,58 @@
-import math
-import time
-
-import mmh3
-import redis
-from cobweb import setting
-
-
-class BloomFilter:
-
-    def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
-        redis_config = redis_config or setting.REDIS_CONFIG
-        capacity = capacity or setting.CAPACITY
-        error_rate = error_rate or setting.ERROR_RATE
-        redis_config['db'] = 3
-
-        self.key = key
-
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
-        self.bit_size = self.get_bit_size(capacity, error_rate)
-        self.hash_count = self.get_hash_count(self.bit_size, capacity)
-        self._init_bloom_key()
-
-    def add(self, value):
-        for seed in range(self.hash_count):
-            result = mmh3.hash(value, seed) % self.bit_size
-            self._client.setbit(self.key, result, 1)
-        return True
-
-    def exists(self, value):
-        if not self._client.exists(self.key):
-            return False
-        for seed in range(self.hash_count):
-            result = mmh3.hash(value, seed) % self.bit_size
-            if not self._client.getbit(self.key, result):
-                return False
-        return True
-
-    def _init_bloom_key(self):
-        lua_script = """
-        redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
-        redis.call("EXPIRE", KEYS[1], 604800)
-        """
-        if self._client.exists(self.key):
-            return True
-        execute = self._client.register_script(lua_script)
-        execute(keys=[self.key], args=[self.bit_size-1, 1])
-
-    @classmethod
-    def get_bit_size(cls, n, p):
-        return int(-(n * math.log(p)) / (math.log(2) ** 2))
-
-    @classmethod
-    def get_hash_count(cls, m, n):
-        return int((m / n) * math.log(2))
-
-
+# import math
+# import time
+#
+# import mmh3
+# import redis
+# from cobweb import setting
+#
+#
+# class BloomFilter:
+#
+#     def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
+#         redis_config = redis_config or setting.REDIS_CONFIG
+#         capacity = capacity or setting.CAPACITY
+#         error_rate = error_rate or setting.ERROR_RATE
+#         redis_config['db'] = 3
+#
+#         self.key = key
+#
+#         pool = redis.ConnectionPool(**redis_config)
+#         self._client = redis.Redis(connection_pool=pool)
+#         self.bit_size = self.get_bit_size(capacity, error_rate)
+#         self.hash_count = self.get_hash_count(self.bit_size, capacity)
+#         self._init_bloom_key()
+#
+#     def add(self, value):
+#         for seed in range(self.hash_count):
+#             result = mmh3.hash(value, seed) % self.bit_size
+#             self._client.setbit(self.key, result, 1)
+#         return True
+#
+#     def exists(self, value):
+#         if not self._client.exists(self.key):
+#             return False
+#         for seed in range(self.hash_count):
+#             result = mmh3.hash(value, seed) % self.bit_size
+#             if not self._client.getbit(self.key, result):
+#                 return False
+#         return True
+#
+#     def _init_bloom_key(self):
+#         lua_script = """
+#         redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
+#         redis.call("EXPIRE", KEYS[1], 604800)
+#         """
+#         if self._client.exists(self.key):
+#             return True
+#         execute = self._client.register_script(lua_script)
+#         execute(keys=[self.key], args=[self.bit_size-1, 1])
+#
+#     @classmethod
+#     def get_bit_size(cls, n, p):
+#         return int(-(n * math.log(p)) / (math.log(2) ** 2))
+#
+#     @classmethod
+#     def get_hash_count(cls, m, n):
+#         return int((m / n) * math.log(2))
+#
+#

cobweb/{base → utils}/decorators.py

@@ -1,16 +1,6 @@
+import time
 from functools import wraps
-
-
-# def check_redis_status(func):
-#     @wraps(func)
-#     def wrapper(*args, **kwargs):
-#         try:
-#             result = func(*args, **kwargs)
-#         except Exception:
-#             result = False
-#         return result
-#
-#     return wrapper
+from cobweb.base import logger
 
 
 def decorator_oss_db(exception, retries=3):
@@ -37,4 +27,16 @@ def decorator_oss_db(exception, retries=3):
     return decorator
 
 
+def check_pause(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        while not self.pause.is_set():
+            try:
+                func(self, *args, **kwargs)
+            except Exception as e:
+                logger.info(f"{func.__name__}: " + str(e))
+            finally:
+                time.sleep(0.1)
+        logger.info(f"Pause detected: {func.__name__} thread closing...")
 
+    return wrapper