cobweb-launcher 0.1.8__py3-none-any.whl → 1.2.41__py3-none-any.whl

cobweb/launchers/launcher_pro.py

@@ -0,0 +1,208 @@
+import time
+import threading
+
+from cobweb.db import RedisDB
+from cobweb.base import Seed, logger
+from cobweb.utils import BloomFilter
+from cobweb.constant import DealModel, LogTemplate
+from .launcher import Launcher, check_pause
+
+
+class LauncherPro(Launcher):
+
+    def __init__(self, task, project, custom_setting=None, **kwargs):
+        super().__init__(task, project, custom_setting, **kwargs)
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._done_key = "{%s:%s}:done" % (project, task)
+        self._fail_key = "{%s:%s}:fail" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+
+        self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
+        self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+
+        # self._bf_key = "bloom_%s_%s" % (project, task)
+        #
+        self._db = RedisDB()
+        #
+        # self._bf = BloomFilter(self._bf_key)
+
+        self._heartbeat_start_event = threading.Event()
+        self._redis_queue_empty_event = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat_key)
+
+    def statistics(self, key, count):
+        if not self._task_model and not self._db.exists(key):
+            self._db.setex(key, 86400 * 30, int(count))
+        else:
+            self._db._client.incrby(key, count)
+
+    def _get_seed(self) -> Seed:
+        spider_speed = self._db._client.get(self._speed_control_key)
+        if int(spider_speed or 0) > self._spider_max_count:
+            expire_time = self._db.ttl(self._speed_control_key)
+            if expire_time == -1:
+                self._db.delete(self._speed_control_key)
+            else:
+                logger.info(f"Too fast! Please wait {expire_time} seconds...")
+                time.sleep(expire_time / 2)
+            return None
+        seed = self.__LAUNCHER_QUEUE__["todo"].pop()
+        if seed and not self._db.lock(self._speed_control_key, t=self._time_window):
+            self._db._client.incrby(self._speed_control_key, 1)
+        return seed
+
+    @check_pause
+    def _execute_heartbeat(self):
+        if self._heartbeat_start_event.is_set():
+            self._db.setex(self._heartbeat_key, 5)
+        time.sleep(3)
+
+    @check_pause
+    def _reset(self):
+        """
+        检查过期种子，重新添加到redis缓存中
+        """
+        reset_wait_seconds = 30
+        if self._db.lock(self._reset_lock_key, t=120):
+
+            _min = -int(time.time()) + self._seed_reset_seconds \
+                if self.heartbeat else "-inf"
+
+            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
+            self._db.delete(self._reset_lock_key)
+
+            if not self.heartbeat:
+                self._heartbeat_start_event.set()
+
+        time.sleep(reset_wait_seconds)
+
+    @check_pause
+    def _scheduler(self):
+        """
+        调度任务，获取redis队列种子，同时添加到doing字典中
+        """
+        if not self._db.zcount(self._todo_key, 0, "(1000"):
+            time.sleep(self._scheduler_wait_seconds)
+        elif self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
+            time.sleep(self._todo_queue_full_wait_seconds)
+        else:
+            members = self._db.members(
+                self._todo_key, int(time.time()),
+                count=self._todo_queue_size,
+                _min=0, _max="(1000"
+            )
+            for member, priority in members:
+                seed = Seed(member, priority=priority)
+                self.__LAUNCHER_QUEUE__['todo'].push(seed)
+                self.__DOING__[seed.to_string] = seed.params.priority
+
+    @check_pause
+    def _insert(self):
+        """
+        添加新种子到redis队列中
+        """
+        seeds = {}
+        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+        for _ in range(self._new_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['new'].pop()
+            if seed:
+                seeds[seed.to_string] = seed.params.priority
+        if seeds:
+            self._db.zadd(self._todo_key, seeds, nx=True)
+        if status:
+            time.sleep(self._new_queue_wait_seconds)
+
+    @check_pause
+    def _refresh(self):
+        """
+        刷新doing种子过期时间，防止reset重新消费
+        """
+        if self.__DOING__:
+            refresh_time = int(time.time())
+            seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
+            self._db.zadd(self._todo_key, item=seeds, xx=True)
+        time.sleep(15)
+
+    @check_pause
+    def _delete(self):
+        """
+        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
+        """
+        seed_list = []
+        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+
+        for _ in range(self._done_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['done'].pop()
+            if not seed:
+                break
+            seed_list.append(seed.to_string)
+
+        if seed_list:
+
+            self._db.zrem(self._todo_key, *seed_list)
+            self._remove_doing_seeds(seed_list)
+
+        if status:
+            time.sleep(self._done_queue_wait_seconds)
+
+    def _polling(self):
+        wait_scheduler_execute = True
+        check_emtpy_times = 0
+        while not self._stop.is_set():
+            queue_not_empty_count = 0
+            pooling_wait_seconds = 30
+
+            for q in self.__LAUNCHER_QUEUE__.values():
+                if q.length != 0:
+                    queue_not_empty_count += 1
+                    wait_scheduler_execute = False
+
+            if queue_not_empty_count == 0:
+                pooling_wait_seconds = 3
+                if self._pause.is_set():
+                    check_emtpy_times = 0
+                    if not self._task_model and (
+                            not wait_scheduler_execute or
+                            int(time.time()) - self._app_time > self._before_scheduler_wait_seconds
+                    ):
+                        logger.info("Done! ready to close thread...")
+                        self._stop.set()
+
+                    elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
+                        logger.info(f"Recovery {self.task} task run！")
+                        self._pause.clear()
+                        self._execute()
+                    else:
+                        logger.info("pause! waiting for resume...")
+                elif check_emtpy_times > 2:
+                    self.__DOING__ = {}
+                    if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
+                        self._pause.set()
+                else:
+                    logger.info(
+                        "check whether the task is complete, "
+                        f"reset times {3 - check_emtpy_times}"
+                    )
+                    check_emtpy_times += 1
+            else:
+                logger.info(LogTemplate.launcher_pro_polling.format(
+                    task=self.task,
+                    doing_len=len(self.__DOING__.keys()),
+                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
+                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
+                    redis_seed_count=self._db.zcount(self._todo_key, "-inf", "+inf"),
+                    redis_todo_len=self._db.zcount(self._todo_key, 0, "(1000"),
+                    redis_doing_len=self._db.zcount(self._todo_key, "-inf", "(0"),
+                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
+                ))
+
+            time.sleep(pooling_wait_seconds)
+
+        logger.info("Done! Ready to close thread...")
+

cobweb/pipelines/__init__.py

@@ -0,0 +1,3 @@
+from .pipeline import Pipeline
+from .pipeline_console import Console
+from .pipeline_loghub import Loghub

cobweb/pipelines/pipeline.py

@@ -0,0 +1,69 @@
+import time
+import threading
+
+from abc import ABC, abstractmethod
+from cobweb.base import BaseItem, Queue, logger
+
+
+class Pipeline(threading.Thread, ABC):
+
+    def __init__(
+            self,
+            stop: threading.Event,
+            pause: threading.Event,
+            upload: Queue, done: Queue,
+            upload_size: int,
+            wait_seconds: int
+    ):
+        super().__init__()
+        self._stop = stop
+        self._pause = pause
+        self._upload = upload
+        self._done = done
+
+        self.upload_size = upload_size
+        self.wait_seconds = wait_seconds
+
+    @abstractmethod
+    def build(self, item: BaseItem) -> dict:
+        pass
+
+    @abstractmethod
+    def upload(self, table: str, data: list) -> bool:
+        pass
+
+    def run(self):
+        while not self._stop.is_set():
+            if not self._upload.length:
+                time.sleep(self.wait_seconds)
+                continue
+            if self._upload.length < self.upload_size:
+                time.sleep(self.wait_seconds)
+            status = True
+            data_info, seeds = {}, []
+            try:
+                for _ in range(self.upload_size):
+                    item = self._upload.pop()
+                    if not item:
+                        break
+                    seeds.append(item.seed)
+                    data = self.build(item)
+                    data_info.setdefault(item.table, []).append(data)
+                for table, datas in data_info.items():
+                    try:
+                        self.upload(table, datas)
+                    except Exception as e:
+                        logger.info(e)
+                        status = False
+            except Exception as e:
+                logger.info(e)
+                status = False
+            if not status:
+                for seed in seeds:
+                    seed.params.seed_status = "deal model: fail"
+            if seeds:
+                self._done.push(seeds)
+
+        logger.info("upload pipeline close!")
+
+

cobweb/pipelines/pipeline_console.py

@@ -0,0 +1,22 @@
+from cobweb.base import ConsoleItem, logger
+from cobweb.constant import LogTemplate
+from cobweb.pipelines import Pipeline
+
+
+class Console(Pipeline):
+
+    def build(self, item: ConsoleItem):
+        return {
+            "seed": item.seed.to_dict,
+            "data": item.to_dict
+        }
+
+    def upload(self, table, datas):
+        for data in datas:
+            parse_detail = LogTemplate.log_info(data["data"])
+            if len(parse_detail) > 500:
+                parse_detail = parse_detail[:500] + " ...\n" + " " * 12 + "-- Text is too long and details are omitted!"
+            logger.info(LogTemplate.console_item.format(
+                seed_detail=LogTemplate.log_info(data["seed"]),
+                parse_detail=parse_detail
+            ))

cobweb/pipelines/pipeline_loghub.py

@@ -0,0 +1,34 @@
+import json
+
+from cobweb import setting
+from cobweb.base import BaseItem
+from cobweb.pipelines import Pipeline
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+
+class Loghub(Pipeline):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = LogClient(**setting.LOGHUB_CONFIG)
+
+    def build(self, item: BaseItem):
+        log_item = LogItem()
+        temp = item.to_dict
+        for key, value in temp.items():
+            if not isinstance(value, str):
+                temp[key] = json.dumps(value, ensure_ascii=False)
+        contents = sorted(temp.items())
+        log_item.set_contents(contents)
+        return log_item
+
+    def upload(self, table, datas):
+        request = PutLogsRequest(
+            project=setting.LOGHUB_PROJECT,
+            logstore=table,
+            topic=setting.LOGHUB_TOPIC,
+            source=setting.LOGHUB_SOURCE,
+            logitems=datas,
+            compress=True
+        )
+        self.client.put_logs(request=request)

cobweb/schedulers/__init__.py

@@ -0,0 +1,3 @@
+from .scheduler_redis import RedisScheduler
+from .scheduler_api import ApiScheduler
+

cobweb/schedulers/scheduler_api.py

@@ -0,0 +1,72 @@
+import threading
+import time
+
+# from cobweb.base import Seed
+from cobweb.db import ApiDB
+
+
+class ApiScheduler:
+
+    def __init__(self, task, project, scheduler_wait_seconds=30):
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._download_key = "{%s:%s}:download" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+        self._db = ApiDB()
+
+        self.scheduler_wait_seconds = scheduler_wait_seconds
+        self.working = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat_key)
+
+    def set_heartbeat(self):
+        return self._db.setex(self._heartbeat_key, 5)
+
+    def schedule(self, key, count):
+        if not self._db.zcount(key, 0, "(1000"):
+            time.sleep(self.scheduler_wait_seconds)
+        else:
+            source = int(time.time())
+            members = self._db.members(key, source, count=count, _min=0, _max="(1000")
+            for member, priority in members:
+                # seed = Seed(member, priority=priority)
+                yield member, priority
+
+    def insert(self, key, items):
+        if items:
+            self._db.zadd(key, items, nx=True)
+
+    def reset(self, keys, reset_time=30):
+        if self._db.lock(self._reset_lock_key, t=120):
+
+            if isinstance(keys, str):
+                keys = [keys]
+
+            _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
+
+            for key in keys:
+                if self._db.exists(key):
+                    self._db.members(key, 0, _min=_min, _max="(0")
+
+            if not self.heartbeat:
+                self.working.set()
+                time.sleep(10)
+
+            self._db.delete(self._reset_lock_key)
+
+    def refresh(self, key, items: dict[str, int]):
+        refresh_time = int(time.time())
+        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
+        if its:
+            self._db.zadd(key, item=its, xx=True)
+
+    def delete(self, key, values):
+        if values:
+            self._db.zrem(key, *values)
+
+
+
+

cobweb/schedulers/scheduler_redis.py

@@ -0,0 +1,72 @@
+import threading
+import time
+
+# from cobweb.base import Seed
+from cobweb.db import RedisDB
+
+
+class RedisScheduler:
+
+    def __init__(self, task, project, scheduler_wait_seconds=30):
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._download_key = "{%s:%s}:download" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+        self._db = RedisDB()
+
+        self.scheduler_wait_seconds = scheduler_wait_seconds
+        self.working = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat_key)
+
+    def set_heartbeat(self):
+        return self._db.setex(self._heartbeat_key, 5)
+
+    def schedule(self, key, count):
+        if not self._db.zcount(key, 0, "(1000"):
+            time.sleep(self.scheduler_wait_seconds)
+        else:
+            source = int(time.time())
+            members = self._db.members(key, source, count=count, _min=0, _max="(1000")
+            for member, priority in members:
+                # seed = Seed(member, priority=priority)
+                yield member, priority
+
+    def insert(self, key, items):
+        if items:
+            self._db.zadd(key, items, nx=True)
+
+    def reset(self, keys, reset_time=30):
+        if self._db.lock(self._reset_lock_key, t=120):
+
+            if isinstance(keys, str):
+                keys = [keys]
+
+            _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
+
+            for key in keys:
+                if self._db.exists(key):
+                    self._db.members(key, 0, _min=_min, _max="(0")
+
+            if not self.heartbeat:
+                self.working.set()
+                time.sleep(10)
+
+            self._db.delete(self._reset_lock_key)
+
+    def refresh(self, key, items: dict[str, int]):
+        refresh_time = int(time.time())
+        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
+        if its:
+            self._db.zadd(key, item=its, xx=True)
+
+    def delete(self, key, values):
+        if values:
+            self._db.zrem(key, *values)
+
+
+
+

cobweb/setting.py

@@ -1,13 +1,74 @@
 import os
 
+# redis db config
+REDIS_CONFIG = {
+    "host": os.getenv("REDIS_HOST"),
+    "password": os.getenv("REDIS_PASSWORD"),
+    "port": int(os.getenv("REDIS_PORT", 6379)),
+    "db": int(os.getenv("REDIS_DB", 0)),
+}
 
-# model: 0, 1, 2
-MODEL = int(os.getenv("MODEL", "0"))
+# loghub db config
+LOGHUB_TOPIC = os.getenv("LOGHUB_TOPIC")
+LOGHUB_SOURCE = os.getenv("LOGHUB_SOURCE")
+LOGHUB_PROJECT = os.getenv("LOGHUB_PROJECT")
+LOGHUB_CONFIG = {
+    "endpoint": os.getenv("LOGHUB_ENDPOINT"),
+    "accessKeyId": os.getenv("LOGHUB_ACCESS_KEY"),
+    "accessKey": os.getenv("LOGHUB_SECRET_KEY")
+}
 
-# 重制score值的等待时间, 默认10分钟
-RESET_SCORE = int(os.getenv("RESET_SCORE", "600"))
+# oss util config
+OSS_BUCKET = os.getenv("OSS_BUCKET")
+OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
+OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
+OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
+OSS_CHUNK_SIZE = 10 * 1024 ** 2
+OSS_MIN_UPLOAD_SIZE = 1024
 
-# 默认设置检查spider queue队列锁的存活时间为30s
-CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
 
+# 采集器选择
+CRAWLER = "cobweb.crawlers.Crawler"
 
+# 数据存储链路
+PIPELINE = "cobweb.pipelines.pipeline_console.Console"
+
+
+# Launcher 等待时间
+
+BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次任务
+SCHEDULER_WAIT_SECONDS = 15  # 调度等待时间
+TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
+NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
+DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
+UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
+SEED_RESET_SECONDS = 30   # 种子重制时间
+
+
+# Launcher 队列长度
+TODO_QUEUE_SIZE = 100  # todo队列长度
+NEW_QUEUE_MAX_SIZE = 100  # new队列长度
+DONE_QUEUE_MAX_SIZE = 100  # done队列长度
+UPLOAD_QUEUE_MAX_SIZE = 100  # upload队列长度
+
+# DONE_MODEL IN (0, 1), 种子完成模式
+DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加至失败队列；1:种子消费成功添加至成功队列，失败添加至失败队列
+
+# spider
+SPIDER_THREAD_NUM = 10
+SPIDER_MAX_RETRIES = 5
+SPIDER_TIME_SLEEP = 10
+
+SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
+TIME_WINDOW = 60   # 频控固定时间窗口（秒）
+
+# 任务模式
+TASK_MODEL = 0  # 0:单次，1:常驻
+
+
+# bloom过滤器
+CAPACITY = 100000000
+ERROR_RATE = 0.001
+FILTER_FIELD = "url"
+# 文件下载响应类型过滤
+# FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]

cobweb/utils/__init__.py

@@ -0,0 +1,5 @@
+from .oss import OssUtil
+from .tools import *
+from .bloom import BloomFilter
+from .dotting import LoghubDot
+

cobweb/utils/bloom.py

@@ -0,0 +1,58 @@
+import math
+import time
+
+import mmh3
+import redis
+from cobweb import setting
+
+
+class BloomFilter:
+
+    def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
+        redis_config = redis_config or setting.REDIS_CONFIG
+        capacity = capacity or setting.CAPACITY
+        error_rate = error_rate or setting.ERROR_RATE
+        redis_config['db'] = 3
+
+        self.key = key
+
+        pool = redis.ConnectionPool(**redis_config)
+        self._client = redis.Redis(connection_pool=pool)
+        self.bit_size = self.get_bit_size(capacity, error_rate)
+        self.hash_count = self.get_hash_count(self.bit_size, capacity)
+        self._init_bloom_key()
+
+    def add(self, value):
+        for seed in range(self.hash_count):
+            result = mmh3.hash(value, seed) % self.bit_size
+            self._client.setbit(self.key, result, 1)
+        return True
+
+    def exists(self, value):
+        if not self._client.exists(self.key):
+            return False
+        for seed in range(self.hash_count):
+            result = mmh3.hash(value, seed) % self.bit_size
+            if not self._client.getbit(self.key, result):
+                return False
+        return True
+
+    def _init_bloom_key(self):
+        lua_script = """
+        redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
+        redis.call("EXPIRE", KEYS[1], 604800)
+        """
+        if self._client.exists(self.key):
+            return True
+        execute = self._client.register_script(lua_script)
+        execute(keys=[self.key], args=[self.bit_size-1, 1])
+
+    @classmethod
+    def get_bit_size(cls, n, p):
+        return int(-(n * math.log(p)) / (math.log(2) ** 2))
+
+    @classmethod
+    def get_hash_count(cls, m, n):
+        return int((m / n) * math.log(2))
+
+

cobweb/utils/dotting.py

@@ -0,0 +1,32 @@
+import json
+
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+from cobweb import setting
+
+
+class LoghubDot:
+
+    def __init__(self):
+        self.client = LogClient(**setting.LOGHUB_CONFIG)
+
+    def build(self, topic, **kwargs):
+
+        temp = {}
+        log_items = []
+        log_item = LogItem()
+        for key, value in kwargs.items():
+            if not isinstance(value, str):
+                temp[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                temp[key] = value
+        contents = sorted(temp.items())
+        log_item.set_contents(contents)
+        log_items.append(log_item)
+        request = PutLogsRequest(
+            project="databee-download-log",
+            logstore="log",
+            topic=topic,
+            logitems=log_items,
+            compress=True
+        )
+        self.client.put_logs(request=request)