cobweb-launcher 1.0.5__py3-none-any.whl → 3.2.18__py3-none-any.whl

cobweb/schedulers/scheduler_with_redis.py

@@ -0,0 +1,189 @@
+import os
+import time
+import threading
+from typing import Callable
+from cobweb.db import RedisDB, ApiDB
+from cobweb.utils import check_pause
+from cobweb.base import Seed, logger, TaskQueue, Status
+from cobweb.constant import LogTemplate
+from .scheduler import Scheduler
+use_api = bool(os.getenv("REDIS_API_HOST", 0))
+
+
+class RedisScheduler(Scheduler):
+
+    def __init__(
+            self,
+            task,
+            project,
+            stop: threading.Event,
+            pause: threading.Event,
+            task_queue: TaskQueue,
+            callback_register: Callable
+    ):
+        super().__init__(task, project, stop, pause, task_queue, callback_register)
+        self.todo_key = f"{{{project}:{task}}}:todo"
+        self.done_key = f"{{{project}:{task}}}:done"
+        self.fail_key = f"{{{project}:{task}}}:fail"
+        self.heartbeat_key = f"heartbeat:{project}_{task}"
+        self.heartbeat_run_key = f"run:{project}_{task}"
+        self.speed_control_key = f"speed_control:{project}_{task}"
+        self.reset_lock_key = f"lock:reset:{project}_{task}"
+        self.db = ApiDB() if use_api else RedisDB()
+
+    def reset(self):
+        """
+        检查过期种子，重新添加到redis缓存中
+        """
+        while not self.stop.is_set():
+            if self.db.lock(self.reset_lock_key, t=360):
+
+                _min = -int(time.time()) + self.seed_reset_seconds
+                self.db.members(self.todo_key, 0, _min=_min, _max="(0")
+                self.db.delete(self.reset_lock_key)
+
+            time.sleep(self.seed_reset_seconds)
+
+    @check_pause
+    def schedule(self):
+        """
+        调度任务，获取redis队列种子，同时添加到doing字典中
+        """
+        if not self.db.zcount(self.todo_key, 0, "(1000"):
+            time.sleep(self.scheduler_wait_seconds)
+            return
+
+        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size\
+                or self.task_queue.length() > 5 * self.todo_queue_size:
+            time.sleep(self.todo_queue_full_wait_seconds)
+            return
+
+        if members := self.db.members(
+            self.todo_key, int(time.time()),
+            count=self.todo_queue_size,
+            _min=0, _max="(1000"
+        ):
+            for member, priority in members:
+                seed = Seed(member, priority=int(priority % 1000))
+                seed.params.get_time = time.time()
+                self.task_queue.add_task(
+                    task_id=seed.sid, data=seed,
+                    status=Status.PENDING,
+                    priority=seed.params.priority
+                )
+
+    @check_pause
+    def insert(self):
+        """
+        添加新种子到redis队列中
+        """
+        if task_list := self.task_queue.get_task_by_status(
+            status=Status.INSERT, limit=self.new_queue_max_size
+        ):
+            seed_info, task_ids = dict(), set()
+
+            for task_item in task_list:
+                seed = task_item.data
+                task_ids.add(task_item.task_id)
+                seed_info[seed.to_string] = seed.params.priority
+
+            self.db.zadd(self.todo_key, seed_info, nx=True)
+            self.task_queue.remove(task_ids)
+
+        if self.task_queue.status_length(status=Status.INSERT) < self.new_queue_max_size:
+            time.sleep(self.scheduler_wait_seconds)
+
+    @check_pause
+    def refresh(self):
+        """
+        刷新doing种子过期时间，防止reset重新消费
+        """
+        if task_list := self.task_queue.get_task_by_status(
+            status=[Status.PENDING, Status.PROCESSING, Status.FINISHED],
+        ):
+            refresh_time = int(time.time())
+            seed_info = {it.data.to_string: -refresh_time - it.data.params.priority / 1000 for it in task_list}
+            self.db.zadd(self.todo_key, seed_info, xx=True)
+        time.sleep(self.seed_reset_seconds // 3)
+
+    @check_pause
+    def delete(self):
+        """
+        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
+        """
+        if task_list := self.task_queue.get_task_by_status(
+                status=Status.FINISHED, limit=self.done_queue_max_size
+        ):
+            zrem_items = [it.data.to_string for it in task_list]
+            remove_task_ids = [it.task_id for it in task_list]
+            self.db.zrem(self.todo_key, *zrem_items)
+            self.task_queue.remove(remove_task_ids)
+
+        if self.task_queue.status_length(status=Status.FINISHED) < self.done_queue_max_size:
+            time.sleep(self.done_queue_wait_seconds)
+
+    def run(self):
+        start_time = int(time.time())
+
+        for func in [self.reset, self.insert, self.delete, self.refresh, self.schedule]:
+            self.callback_register(func, tag="scheduler")
+
+        while not self.stop.is_set():
+            todo_len = self.task_queue.status_length(status=Status.PENDING)
+            doing_len = self.task_queue.status_length(status=Status.PROCESSING)
+            done_len = self.task_queue.status_length(status=Status.FINISHED)
+            upload_len = self.task_queue.status_length(status=Status.UPLOAD)
+
+            redis_doing_count = self.db.zcount(self.todo_key, "-inf", "(0")
+            redis_todo_len = self.db.zcount(self.todo_key, 0, "(1000")
+            redis_seed_count = self.db.zcard(self.todo_key)
+
+            if self.pause.is_set():
+                execute_time = int(time.time()) - start_time
+                if not self.task_model and execute_time > self.before_scheduler_wait_seconds:
+                    logger.info("Done! ready to close thread...")
+                    self.stop.set()
+                elif redis_todo_len:
+                    logger.info(
+                        f"Recovery {self.task} task run！"
+                        f"Todo seeds count: {redis_todo_len}"
+                        f", queue length: {redis_seed_count}"
+                    )
+                    self.pause.clear()
+                else:
+                    logger.info("Pause! waiting for resume...")
+
+            elif self.task_queue.length() == 0:
+                if redis_seed_count:
+                    logger.info(
+                        f"Todo seeds count: {redis_todo_len}"
+                        f", queue length: {redis_seed_count}"
+                    )
+                    self.pause.clear()
+                else:
+                    count = 0
+                    for _ in range(3):
+                        if not redis_seed_count:
+                            count += 1
+                            time.sleep(5)
+                            logger.info("Checking count...")
+                        else:
+                            break
+                    if count >= 3:
+                        logger.info("Todo queue is empty! Pause set...")
+                        self.pause.set()
+
+            else:
+                self.db.setex(self.heartbeat_run_key, 60, 1)
+                logger.info(LogTemplate.launcher_pro_polling.format(
+                    task=self.task,
+                    doing_len=doing_len,
+                    todo_len=todo_len,
+                    done_len=done_len,
+                    redis_seed_count=redis_seed_count,
+                    redis_todo_len=redis_todo_len,
+                    redis_doing_len=redis_doing_count,
+                    upload_len=upload_len,
+                ))
+
+            time.sleep(30)

cobweb/setting.py

@@ -1,45 +1,22 @@
-import os
-
-# redis db config
-REDIS_CONFIG = {
-    "host": os.getenv("REDIS_HOST"),
-    "password": os.getenv("REDIS_PASSWORD"),
-    "port": int(os.getenv("REDIS_PORT", 6379)),
-    "db": int(os.getenv("REDIS_DB", 0)),
-}
-
-# loghub db config
-LOGHUB_TOPIC = os.getenv("LOGHUB_TOPIC")
-LOGHUB_SOURCE = os.getenv("LOGHUB_SOURCE")
-LOGHUB_PROJECT = os.getenv("LOGHUB_PROJECT")
-LOGHUB_CONFIG = {
-    "endpoint": os.getenv("LOGHUB_ENDPOINT"),
-    "accessKeyId": os.getenv("LOGHUB_ACCESS_KEY"),
-    "accessKey": os.getenv("LOGHUB_SECRET_KEY")
-}
-
-# oss util config
-OSS_BUCKET = os.getenv("OSS_BUCKET")
-OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
-OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
-OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
-OSS_MIN_UPLOAD_SIZE = 1024 * 100
-OSS_CHUNK_SIZE = 1024 ** 2
-
 # 采集器选择
-CRAWLER = "cobweb.crawlers.CrawlerAir"
+CRAWLER = "cobweb.crawlers.Crawler"
 
-# 数据上传链路
-PIPELINE = "cobweb.pipelines.loghub_pipeline.LoghubPipeline"
+# 数据管道
+PIPELINE = "cobweb.pipelines.CSV"
+
+# 调度器
+SCHEDULER = "cobweb.schedulers.RedisScheduler"
 
 
 # Launcher 等待时间
+
+BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次任务
 SCHEDULER_WAIT_SECONDS = 15  # 调度等待时间
 TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
-DONE_QUEUE_WAIT_SECONDS = 15   # done队列等待时间
+DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
 UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
-SEED_RESET_SECONDS = 600   # 种子重制时间
+SEED_RESET_SECONDS = 60   # 种子重制时间
 
 
 # Launcher 队列长度
@@ -51,12 +28,34 @@ UPLOAD_QUEUE_MAX_SIZE = 100  # upload队列长度
 # DONE_MODEL IN (0, 1), 种子完成模式
 DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加至失败队列；1:种子消费成功添加至成功队列，失败添加至失败队列
 
-# DOWNLOAD_MODEL IN (0, 1), 下载模式
-DOWNLOAD_MODEL = 0  # 0: 通用下载；1:文件下载
-
 # spider
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
+SPIDER_TIME_SLEEP = 10
+RECORD_FAILED_SPIDER = True
+
+SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
+TIME_WINDOW = 60   # 频控固定时间窗口（秒）
+
+# 任务模式, 0:单次，1:常驻
+TASK_MODEL = 0
+
+# 流控措施, 0:关闭，1:开启
+SPEED_CONTROL = 1
+
+DOT = 0
+
+# redis config
+REDIS_CONFIG = {
+    "host": "127.0.0.1",
+    "port": 6379,
+    "db": 0
+}
 
-# 文件下载响应类型过滤
-FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]
+# loghub pipeline config
+# os.getenv("LOGHUB_ENDPOINT"),
+# os.getenv("LOGHUB_ACCESS_KEY"),
+# os.getenv("LOGHUB_SECRET_KEY")
+# os.getenv("LOGHUB_PROJECT")
+# os.getenv("LOGHUB_SOURCE")
+# os.getenv("LOGHUB_TOPIC")

cobweb/utils/__init__.py

@@ -1,3 +1,6 @@
-from .oss import OssUtil
-from .tools import *
+# from .oss import OssUtil
+# from .bloom import BloomFilter
+# from .dotting import LoghubDot
+from .decorators import check_pause
+from .tools import md5, dynamic_load_class
 

cobweb/utils/bloom.py

@@ -0,0 +1,58 @@
+# import math
+# import time
+#
+# import mmh3
+# import redis
+# from cobweb import setting
+#
+#
+# class BloomFilter:
+#
+#     def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
+#         redis_config = redis_config or setting.REDIS_CONFIG
+#         capacity = capacity or setting.CAPACITY
+#         error_rate = error_rate or setting.ERROR_RATE
+#         redis_config['db'] = 3
+#
+#         self.key = key
+#
+#         pool = redis.ConnectionPool(**redis_config)
+#         self._client = redis.Redis(connection_pool=pool)
+#         self.bit_size = self.get_bit_size(capacity, error_rate)
+#         self.hash_count = self.get_hash_count(self.bit_size, capacity)
+#         self._init_bloom_key()
+#
+#     def add(self, value):
+#         for seed in range(self.hash_count):
+#             result = mmh3.hash(value, seed) % self.bit_size
+#             self._client.setbit(self.key, result, 1)
+#         return True
+#
+#     def exists(self, value):
+#         if not self._client.exists(self.key):
+#             return False
+#         for seed in range(self.hash_count):
+#             result = mmh3.hash(value, seed) % self.bit_size
+#             if not self._client.getbit(self.key, result):
+#                 return False
+#         return True
+#
+#     def _init_bloom_key(self):
+#         lua_script = """
+#         redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
+#         redis.call("EXPIRE", KEYS[1], 604800)
+#         """
+#         if self._client.exists(self.key):
+#             return True
+#         execute = self._client.register_script(lua_script)
+#         execute(keys=[self.key], args=[self.bit_size-1, 1])
+#
+#     @classmethod
+#     def get_bit_size(cls, n, p):
+#         return int(-(n * math.log(p)) / (math.log(2) ** 2))
+#
+#     @classmethod
+#     def get_hash_count(cls, m, n):
+#         return int((m / n) * math.log(2))
+#
+#

cobweb/{base → utils}/decorators.py

@@ -1,16 +1,6 @@
+import time
 from functools import wraps
-
-
-# def check_redis_status(func):
-#     @wraps(func)
-#     def wrapper(*args, **kwargs):
-#         try:
-#             result = func(*args, **kwargs)
-#         except Exception:
-#             result = False
-#         return result
-#
-#     return wrapper
+from cobweb.base import logger
 
 
 def decorator_oss_db(exception, retries=3):
@@ -37,4 +27,16 @@ def decorator_oss_db(exception, retries=3):
     return decorator
 
 
+def check_pause(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        while not self.pause.is_set():
+            try:
+                func(self, *args, **kwargs)
+            except Exception as e:
+                logger.info(f"{func.__name__}: " + str(e))
+            finally:
+                time.sleep(0.1)
+        logger.info(f"Pause detected: {func.__name__} thread closing...")
 
+    return wrapper

cobweb/utils/dotting.py

@@ -0,0 +1,300 @@
+import os
+import json
+import time
+from threading import Event
+from requests import RequestException, Response as requests_Response
+
+from cobweb.base import Queue, Request, Seed, Response, BaseItem, logger
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+
+class LoghubDot:
+
+    def __init__(self, stop: Event, project: str, task: str) -> None:
+        self._stop = stop
+        self._queue = Queue()
+        self._client = LogClient(
+            endpoint=os.getenv("LOGHUB_ENDPOINT"),
+            accessKeyId=os.getenv("LOGHUB_ACCESS_KEY"),
+            accessKey=os.getenv("LOGHUB_SECRET_KEY")
+        )
+        self.project = project
+        self.task = task
+
+    def logging(self, topic, msg):
+        log_item = LogItem()
+        log_data = {
+            "stage": topic,
+            "message": msg,
+            "project": self.project,
+            "task": self.task,
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_request_log(self, request_item: Request):
+        log_item = LogItem()
+
+        seed: Seed = request_item.seed
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        stage_cost = request_time - start_time
+        cost = request_time - start_time
+
+        request_settings = json.dumps(
+            request_item.request_settings,
+            ensure_ascii=False, separators=(',', ':')
+        )
+
+        log_data = {
+            "stage": "request",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "request": repr(request_item),
+            "request_settings": request_settings,
+            "get_time": get_time,
+            "start_time": start_time,
+            "stage_cost": stage_cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(request_time)),
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_download_log(self, response_item: Response):
+        """
+        构建下载阶段的日志项
+
+        Args:
+            response_item: 响应对象
+        """
+        log_item = LogItem()
+
+        seed: Seed = response_item.seed
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        download_time = seed.params.download_time
+        stage_cost = download_time - request_time
+        cost = download_time - start_time
+
+        log_data = {
+            "stage": "download",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "response": repr(response_item),
+            "get_time": get_time,
+            "start_time": start_time,
+            "request_time": request_time,
+            "download_time": download_time,
+            "stage_cost": stage_cost,
+            "cost": cost,
+            "proxy": seed.params.proxy or '-',
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(download_time)),
+        }
+
+        response = response_item.response
+        if isinstance(response, requests_Response):
+            log_data['request_info'] = {
+                'method': response.request.method,
+                'url': response.request.url,
+                'headers': dict(response.request.headers),
+                'body': response.request.body or "-",
+            }
+            log_data['response_info'] = {
+                "status_code": response.status_code,
+                "reason": response.reason,
+                "headers": dict(response.headers),
+                "content": response.text[:500],  # 截取内容
+                "content_type": response.headers.get('content-type', '-'),
+                "content_length": response.headers.get('content-length', '-'),
+                "server": response.headers.get('server', '-'),
+                "date": response.headers.get('date', '-'),
+            }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_parse_log(self, parse_item: BaseItem):
+        log_item = LogItem()
+
+        seed: Seed = parse_item.seed
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        response_time = seed.params.response_time
+        parse_time = seed.params.parse_time
+
+        pre_time = request_time or response_time
+        stage_cost = parse_time - pre_time
+        cost = parse_time - start_time
+
+        log_data = {
+            "stage": "parse",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "parse": repr(parse_item),
+            "get_time": get_time,
+            "start_time": start_time,
+            "parse_time": parse_time,
+            "stage_cost": stage_cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(parse_time)),
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_http_error_log(self, seed: Seed, e: RequestException):
+        log_item = LogItem()
+
+        status_code = getattr(e.response, 'status_code', '-')
+
+        request_info = {
+            'method': getattr(e.request, 'method', '-'),
+            'url': getattr(e.request, 'url', '-'),
+            'headers': dict(getattr(e.request, 'headers', {})),
+            'body': getattr(e.request, 'body', '-'),
+        }
+
+        response_info = {
+            'status_code': getattr(e.response, 'status_code', '-'),
+            'reason': getattr(e.response, 'reason', '-'),
+            'headers': dict(getattr(e.response, 'headers', {})),
+            'content': getattr(e.response, 'text', '')[:500],
+            'content_type': e.response.headers.get('content-type', '-') if e.response else '-',
+            'content_length': e.response.headers.get('content-length', '-') if e.response else '-',
+            'server': e.response.headers.get('server', '-') if e.response else '-',
+            'date': e.response.headers.get('date', '-') if e.response else '-',
+        }
+        retry = seed.params.retry
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        failed_time = seed.params.failed_time
+        cost = failed_time - start_time
+
+        log_data = {
+            "stage": "http_error",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string, 
+            "status_code": status_code,
+            "request_info": request_info,
+            "response_info": response_info,
+            "retry": retry,
+            "proxy": seed.params.proxy or '-',
+            "exception_type": type(e).__name__,
+            "exception_message": str(e),
+            "traceback": seed.params.traceback or '-',
+            "get_time": get_time,
+            "start_time": start_time,
+            "error_time": failed_time,
+            "stage_cost": cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(failed_time)),
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_exception_log(self, seed: Seed, e: Exception):
+        log_item = LogItem()
+
+        retry = seed.params.retry
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        failed_time = seed.params.failed_time
+        cost = failed_time - start_time
+
+        log_data = {
+            "stage": "exception",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "retry": retry,
+            "exception_type": type(e).__name__,
+            "exception_message": str(e),
+            "traceback": seed.params.traceback or '-',
+            "proxy": seed.params.proxy or '-',
+            "get_time": get_time,
+            "start_time": start_time,
+            "error_time": failed_time,
+            "stage_cost": cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(failed_time)),
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_run(self):
+        while not self._stop.is_set():
+            try:
+                items = []
+                start_time = int(time.time())
+
+                while len(items) < 1000:
+                    log_item = self._queue.pop()
+                    if not log_item or (int(time.time()) - start_time > 10):
+                        break
+                    items.append(log_item)
+
+                if items:
+                    request = PutLogsRequest(
+                        project="databee-download-log",
+                        logstore="log",
+                        topic="cobweb",
+                        logitems=items,
+                        compress=True
+                    )
+                    self._client.put_logs(request=request)
+            except Exception as e:
+                logger.info(str(e))