cobweb-launcher 1.3.5__py3-none-any.whl → 1.3.7__py3-none-any.whl

cobweb_new/launchers/launcher_api.py

@@ -1,161 +0,0 @@
-import time
-import threading
-
-from cobweb.db import ApiDB
-from cobweb.base import Seed, TaskQueue,logger, stop, pause
-from cobweb.constant import DealModel
-from .launcher import Launcher
-
-
-class LauncherApi(Launcher):
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__(task, project, custom_setting, **kwargs)
-        self._db = ApiDB()
-
-        self._todo_key = "{%s:%s}:todo" % (project, task)
-        self._done_key = "{%s:%s}:done" % (project, task)
-        self._fail_key = "{%s:%s}:fail" % (project, task)
-        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
-
-        self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
-        self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
-        self._speed_control_key = "speed_control:%s_%s" % (project, task)
-
-        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
-
-        self._heartbeat_start_event = threading.Event()
-
-    @property
-    def heartbeat(self):
-        return self._db.exists(self._heartbeat_key)
-
-    def statistics(self, key, count):
-        if not self.task_model and not self._db.exists(key):
-            self._db.setex(key, 86400 * 30, int(count))
-        else:
-            self._db.incrby(key, count)
-
-    def _get_seed(self) -> Seed:
-        """
-        从队列中获取种子（频控）
-        设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
-        :return: True -> 种子, False -> None
-        """
-        if TaskQueue.TODO.length and not self._db.auto_incr(
-                self._speed_control_key, 
-                t=self.time_window,
-                limit=self.spider_max_count
-        ):
-            expire_time = self._db.ttl(self._speed_control_key)
-            logger.info(f"Too fast! Please wait {expire_time} seconds...")
-            time.sleep(expire_time / 2)
-            return None
-        return TaskQueue.TODO.pop()
-
-    @stop
-    def _reset(self):
-        """
-        检查过期种子，重新添加到redis缓存中
-        """
-        if self._db.lock(self._reset_lock_key, t=120):
-
-            _min = -int(time.time()) + self.seed_reset_seconds \
-                if self.heartbeat else "-inf"
-
-            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
-
-            if not self.heartbeat:
-                self._heartbeat_start_event.set()
-
-            self._db.delete(self._reset_lock_key)
-
-        time.sleep(30)
-
-    @stop
-    def _refresh(self):
-        """
-        刷新doing种子过期时间，防止reset重新消费
-        """
-        if self.doing_seeds:
-            refresh_time = int(time.time())
-            seeds = {k: -refresh_time - v / 1e3 for k, v in self.doing_seeds.items()}
-            self._db.zadd(self._todo_key, item=seeds, xx=True)
-        time.sleep(3)
-
-    @stop
-    def _scheduler(self):
-        """
-        调度任务，获取redis队列种子，同时添加到doing字典中
-        """
-        if not self._db.zcount(self._todo_key, 0, "(1000"):
-            time.sleep(self.scheduler_wait_seconds)
-        elif TaskQueue.TODO.length >= self.todo_queue_size:
-            time.sleep(self.todo_queue_full_wait_seconds)
-        else:
-            members = self._db.members(
-                self._todo_key, int(time.time()),
-                count=self.todo_queue_size,
-                _min=0, _max="(1000"
-            )
-            for member, priority in members:
-                seed = Seed(member, priority=priority)
-                TaskQueue.TODO.push(seed)
-                self.doing_seeds[seed.to_string] = seed.params.priority
-
-    @pause
-    def _heartbeat(self):
-        if self._heartbeat_start_event.is_set():
-            self._db.setex(self._heartbeat_key, t=5)
-        time.sleep(3)
-
-    @pause
-    def _insert(self):
-        """
-        添加新种子到redis队列中
-        """
-        seeds = {}
-        for _ in range(self.new_queue_max_size):
-            if seed := TaskQueue.SEED.pop():
-                seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self._db.zadd(self._todo_key, seeds, nx=True)
-        if TaskQueue.SEED.length < self.new_queue_max_size:
-            time.sleep(self.new_queue_wait_seconds)
-
-    @pause
-    def _delete(self):
-        """
-        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
-        """
-        seed_info = {"count": 0, "failed": [], "succeed": [], "common": []}
-        status = TaskQueue.DONE.length < self.done_queue_max_size
-
-        for _ in range(self.done_queue_max_size):
-            seed = TaskQueue.DONE.pop()
-            if not seed:
-                break
-            if seed.params.seed_status == DealModel.fail:
-                seed_info["failed"].append(seed.to_string)
-            elif self.done_model == 1:
-                seed_info["succeed"].append(seed.to_string)
-            else:
-                seed_info["common"].append(seed.to_string)
-            seed_info['count'] += 1
-
-        if seed_info["count"]:
-
-            succeed_count = int(self._db.zrem(self._todo_key, *seed_info["common"]) or 0)
-            succeed_count += int(self._db.done([self._todo_key, self._done_key], *seed_info["succeed"]) or 0)
-            failed_count = int(self._db.done([self._todo_key, self._fail_key], *seed_info["failed"]) or 0)
-
-            if failed_count:
-                self.statistics(self._statistics_fail_key, failed_count)
-            if succeed_count:
-                self.statistics(self._statistics_done_key, succeed_count)
-
-            self._remove_doing_seeds(seed_info["common"] + seed_info["succeed"] + seed_info["failed"])
-
-        if status:
-            time.sleep(self.done_queue_wait_seconds)
-

cobweb_new/launchers/launcher_pro.py

@@ -1,96 +0,0 @@
-import time
-
-from base import TaskQueue
-from cobweb.base import decorators
-from schedulers.scheduler_redis import RedisScheduler
-from .launcher import Launcher
-
-
-class LauncherPro(Launcher):
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__(task, project, custom_setting, **kwargs)
-        self._redis_download = "{%s:%s}:download" % (project, task)
-        self._redis_todo = "{%s:%s}:todo" % (project, task)
-        self._scheduler = RedisScheduler(task, project)
-
-    # @decorators.add_thread()
-    @decorators.stop
-    def _schedule(self):
-        thread_sleep = self.scheduling_wait_time
-        for q, key, size in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size),
-        ]:
-            if q.length < size:
-                for item in self._scheduler.schedule(
-                    key, self.scheduling_size
-                ):
-                    q.push(item)
-                thread_sleep = 0.1
-        time.sleep(thread_sleep)
-
-    # @decorators.add_thread()
-    @decorators.pause
-    def _heartbeat(self):
-        if self._scheduler.working.is_set():
-            self._scheduler.set_heartbeat()
-        time.sleep(3)
-
-    # @decorators.add_thread()
-    @decorators.pause
-    def _reset(self):
-        self._scheduler.reset(
-            keys=[self._redis_todo, self._redis_download],
-            reset_time=self.seed_reset_seconds
-        )
-        time.sleep(15)
-
-    # @decorators.add_thread()
-    @decorators.pause
-    def _insert(self):
-        thread_sleep = 0.1
-        for q, key, size in [
-            (TaskQueue.SEED, self._redis_todo, self.seed_queue_size),
-            (TaskQueue.REQUEST, self._redis_download, self.request_queue_size),
-        ]:
-            items = {}
-            while item := q.pop() and len(items.keys()) < self.inserting_size:
-                items[item.to_string] = item.params.priority
-            if q.length >= size:
-                thread_sleep = self.inserting_wait_time
-            self._scheduler.insert(key, items)
-        time.sleep(thread_sleep)
-
-    # @decorators.add_thread()
-    @decorators.pause
-    def _refresh(self):
-        self._scheduler.refresh(self._redis_todo, self._task_info["todo"])
-        self._scheduler.refresh(self._redis_download, self._task_info["download"])
-        time.sleep(3)
-
-    # @decorators.add_thread()
-    @decorators.pause
-    def _remove(self):
-        thread_sleep = self.removing_wait_time
-        for q, key, size in [
-            (TaskQueue.DELETE, self._redis_todo, self.delete_queue_size),
-            (TaskQueue.DONE, self._redis_download, self.done_queue_size),
-        ]:
-            items = []
-            while item := q.pop() and len(items) < self.removing_size:
-                items.append(item)
-            self._scheduler.delete(key, *items)
-            self.remove_working_items(key.split(":")[-1], items)
-            if q.length >= size:
-                thread_sleep = 0.1
-        time.sleep(thread_sleep)
-
-    def _init_schedule_thread(self):
-        self._add_thread(func=self._heartbeat)
-        self._add_thread(func=self._reset)
-        self._add_thread(func=self._refresh)
-        self._add_thread(func=self._schedule)
-        self._add_thread(func=self._insert)
-        self._add_thread(func=self._remove)
-        self._add_thread(func=self._polling)

cobweb_new/launchers/tesss.py

@@ -1,47 +0,0 @@
-import threading
-import time
-from functools import wraps
-
-
-def add_thread(num=1):
-    def decorator(func):
-        @wraps(func)
-        def wrapper(self, *args):
-            for i in range(num):
-                name = func.__name__ + "_" + str(i) if num > 1 else func.__name__
-                self._threads.append(threading.Thread(name=name, target=func, args=(self,) + args))
-        return wrapper
-
-    return decorator
-
-
-def pause(func):
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        while True:
-            try:
-                func(*args, **kwargs)
-            except Exception as e:
-                print(str(e))
-            finally:
-                time.sleep(0.1)
-
-    return wrapper
-
-
-class TTT:
-    _threads = []
-
-    @add_thread()
-    @pause
-    def tt(self):
-        print("hello")
-        time.sleep(1)
-
-tttt = TTT()
-tttt.tt()
-print(TTT._threads)
-
-
-for _ in TTT._threads:
-    _.start()

cobweb_new/pipelines/__init__.py

@@ -1,3 +0,0 @@
-from .pipeline import Pipeline
-from .pipeline_console import Console
-from .pipeline_loghub import Loghub

cobweb_new/pipelines/pipeline.py

@@ -1,68 +0,0 @@
-import time
-import threading
-
-from abc import ABC, abstractmethod
-
-from cobweb.utils import TaskQueue
-from cobweb.base import BaseItem, logger
-
-
-class Pipeline(threading.Thread, ABC):
-
-    def __init__(
-            self,
-            stop: threading.Event,
-            pause: threading.Event,
-            upload_size: int,
-            wait_seconds: int
-    ):
-        super().__init__()
-        self._stop = stop
-        self._pause = pause
-
-        self.upload_size = upload_size
-        self.wait_seconds = wait_seconds
-
-    @abstractmethod
-    def build(self, item: BaseItem) -> dict:
-        pass
-
-    @abstractmethod
-    def upload(self, table: str, data: list) -> bool:
-        pass
-
-    def run(self):
-        while not self._stop.is_set():
-            if not TaskQueue.UPLOAD.length:
-                time.sleep(self.wait_seconds)
-                continue
-            if TaskQueue.UPLOAD.length < self.upload_size:
-                time.sleep(self.wait_seconds)
-            status = True
-            data_info, seeds = {}, []
-            try:
-                for _ in range(self.upload_size):
-                    item = TaskQueue.UPLOAD.pop()
-                    if not item:
-                        break
-                    seeds.append(item.seed)
-                    data = self.build(item)
-                    data_info.setdefault(item.table, []).append(data)
-                for table, datas in data_info.items():
-                    try:
-                        self.upload(table, datas)
-                    except Exception as e:
-                        logger.info(e)
-                        status = False
-            except Exception as e:
-                logger.info(e)
-                status = False
-            if not status:
-                for seed in seeds:
-                    seed.params.seed_status = "deal model: fail"
-            if seeds:
-                TaskQueue.DONE.push(seeds)
-
-        logger.info("upload pipeline close!")
-
-

cobweb_new/pipelines/pipeline_console.py

@@ -1,22 +0,0 @@
-from cobweb.base import ConsoleItem, logger
-from cobweb.constant import LogTemplate
-from cobweb.pipelines import Pipeline
-
-
-class Console(Pipeline):
-
-    def build(self, item: ConsoleItem):
-        return {
-            "seed": item.seed.to_dict,
-            "data": item.to_dict
-        }
-
-    def upload(self, table, datas):
-        for data in datas:
-            parse_detail = LogTemplate.log_info(data["data"])
-            if len(parse_detail) > 500:
-                parse_detail = parse_detail[:500] + " ...\n" + " " * 12 + "-- Text is too long and details are omitted!"
-            logger.info(LogTemplate.console_item.format(
-                seed_detail=LogTemplate.log_info(data["seed"]),
-                parse_detail=parse_detail
-            ))

cobweb_new/pipelines/pipeline_loghub.py

@@ -1,34 +0,0 @@
-import json
-
-from cobweb import setting
-from cobweb.base import BaseItem
-from cobweb.pipelines import Pipeline
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-
-
-class Loghub(Pipeline):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.client = LogClient(**setting.LOGHUB_CONFIG)
-
-    def build(self, item: BaseItem):
-        log_item = LogItem()
-        temp = item.to_dict
-        for key, value in temp.items():
-            if not isinstance(value, str):
-                temp[key] = json.dumps(value, ensure_ascii=False)
-        contents = sorted(temp.items())
-        log_item.set_contents(contents)
-        return log_item
-
-    def upload(self, table, datas):
-        request = PutLogsRequest(
-            project=setting.LOGHUB_PROJECT,
-            logstore=table,
-            topic=setting.LOGHUB_TOPIC,
-            source=setting.LOGHUB_SOURCE,
-            logitems=datas,
-            compress=True
-        )
-        self.client.put_logs(request=request)

cobweb_new/setting.py

@@ -1,95 +0,0 @@
-import os
-
-# redis db config
-REDIS_CONFIG = {
-    "host": os.getenv("REDIS_HOST"),
-    "password": os.getenv("REDIS_PASSWORD"),
-    "port": int(os.getenv("REDIS_PORT", 6379)),
-    "db": int(os.getenv("REDIS_DB", 0)),
-}
-
-# loghub db config
-LOGHUB_TOPIC = os.getenv("LOGHUB_TOPIC")
-LOGHUB_SOURCE = os.getenv("LOGHUB_SOURCE")
-LOGHUB_PROJECT = os.getenv("LOGHUB_PROJECT")
-LOGHUB_CONFIG = {
-    "endpoint": os.getenv("LOGHUB_ENDPOINT"),
-    "accessKeyId": os.getenv("LOGHUB_ACCESS_KEY"),
-    "accessKey": os.getenv("LOGHUB_SECRET_KEY")
-}
-
-# oss util config
-OSS_BUCKET = os.getenv("OSS_BUCKET")
-OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
-OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
-OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
-OSS_CHUNK_SIZE = 10 * 1024 ** 2
-OSS_MIN_UPLOAD_SIZE = 1024
-
-
-# 采集器选择
-CRAWLER = "cobweb.crawlers.Crawler"
-
-# 数据存储链路
-PIPELINE = "cobweb.pipelines.pipeline_console.Console"
-
-
-# Launcher 等待时间
-
-BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次任务
-
-SCHEDULING_WAIT_TIME = 15  # SCHEDULER ITEM 调度等待时间
-INSERTING_WAIT_TIME = 30   # INSERT ITEM 等待时间
-REMOVING_WAIT_TIME = 5     # REMOVE ITEM 等待时间
-
-TODO_QUEUE_FULL_WAIT_SECONDS = 5  # 队列已满时等待时间
-NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
-DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
-UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
-SEED_RESET_SECONDS = 30   # 种子重制时间
-
-
-# Launcher 队列长度
-SCHEDULING_SIZE = 100   # 调度队列长度
-INSERTING_SIZE = 100    # INSERT 长度
-REMOVING_SIZE = 100     # REMOVE 长度
-
-# SEED = Queue()          # 添加任务种子队列
-# TODO = Queue()          # 任务种子队列
-# REQUEST = Queue()       # 请求队列
-# DOWNLOAD = Queue()      # 下载任务队列
-# RESPONSE = Queue()      # 响应队列
-# DONE = Queue()          # 下载完成队列
-# UPLOAD = Queue()        # 任务上传队列
-# DELETE = Queue()        # 任务删除队列
-
-SEED_QUEUE_SIZE = 100       # TODO 队列长度
-TODO_QUEUE_SIZE = 100       # TODO 队列长度
-REQUEST_QUEUE_SIZE = 100    # new队列长度
-DOWNLOAD_QUEUE_SIZE = 100   # done队列长度
-RESPONSE_QUEUE_SIZE = 100   # upload队列长度
-DONE_QUEUE_SIZE = 100       # upload队列长度
-UPLOAD_QUEUE_SIZE = 100     # upload队列长度
-DELETE_QUEUE_SIZE = 100     # upload队列长度
-
-# DONE_MODEL IN (0, 1), 种子完成模式
-DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加至失败队列；1:种子消费成功添加至成功队列，失败添加至失败队列
-
-# spider
-SPIDER_THREAD_NUM = 10
-SPIDER_MAX_RETRIES = 5
-SPIDER_TIME_SLEEP = 10
-
-SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
-TIME_WINDOW = 60   # 频控固定时间窗口（秒）
-
-# 任务模式
-TASK_MODEL = 0  # 0:单次，1:常驻
-
-
-# bloom过滤器
-CAPACITY = 100000000
-ERROR_RATE = 0.001
-FILTER_FIELD = "url"
-# 文件下载响应类型过滤
-# FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]

cobweb_new/utils/__init__.py

@@ -1,5 +0,0 @@
-from .oss import OssUtil
-from .tools import *
-from .bloom import BloomFilter
-from .task_queue import TaskQueue
-

cobweb_new/utils/bloom.py

@@ -1,58 +0,0 @@
-import math
-import time
-
-import mmh3
-import redis
-from cobweb import setting
-
-
-class BloomFilter:
-
-    def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
-        redis_config = redis_config or setting.REDIS_CONFIG
-        capacity = capacity or setting.CAPACITY
-        error_rate = error_rate or setting.ERROR_RATE
-        redis_config['db'] = 3
-
-        self.key = key
-
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
-        self.bit_size = self.get_bit_size(capacity, error_rate)
-        self.hash_count = self.get_hash_count(self.bit_size, capacity)
-        self._init_bloom_key()
-
-    def add(self, value):
-        for seed in range(self.hash_count):
-            result = mmh3.hash(value, seed) % self.bit_size
-            self._client.setbit(self.key, result, 1)
-        return True
-
-    def exists(self, value):
-        if not self._client.exists(self.key):
-            return False
-        for seed in range(self.hash_count):
-            result = mmh3.hash(value, seed) % self.bit_size
-            if not self._client.getbit(self.key, result):
-                return False
-        return True
-
-    def _init_bloom_key(self):
-        lua_script = """
-        redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
-        redis.call("EXPIRE", KEYS[1], 604800)
-        """
-        if self._client.exists(self.key):
-            return True
-        execute = self._client.register_script(lua_script)
-        execute(keys=[self.key], args=[self.bit_size-1, 1])
-
-    @classmethod
-    def get_bit_size(cls, n, p):
-        return int(-(n * math.log(p)) / (math.log(2) ** 2))
-
-    @classmethod
-    def get_hash_count(cls, m, n):
-        return int((m / n) * math.log(2))
-
-

cobweb_new/utils/oss.py

@@ -1,94 +0,0 @@
-from typing import List
-from cobweb import setting
-from requests import Response
-from oss2 import Auth, Bucket, models, PartIterator
-from cobweb.exceptions import oss_db_exception
-from cobweb.base.decorators import decorator_oss_db
-
-
-class OssUtil:
-
-    def __init__(
-        self,
-        bucket=None,
-        endpoint=None,
-        access_key=None,
-        secret_key=None,
-        chunk_size=None,
-        min_upload_size=None,
-        **kwargs
-    ):
-        self.bucket = bucket or setting.OSS_BUCKET
-        self.endpoint = endpoint or setting.OSS_ENDPOINT
-        self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
-        self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
-
-        self._auth = Auth(
-            access_key_id=access_key or setting.OSS_ACCESS_KEY,
-            access_key_secret=secret_key or setting.OSS_SECRET_KEY
-        )
-        self._client = Bucket(
-            auth=self._auth,
-            endpoint=self.endpoint,
-            bucket_name=self.bucket,
-            **kwargs
-        )
-
-    def exists(self, key: str) -> bool:
-        return self._client.object_exists(key)
-
-    def head(self, key: str) -> models.HeadObjectResult:
-        return self._client.head_object(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
-    def init_part(self, key) -> models.InitMultipartUploadResult:
-        """初始化分片上传"""
-        return self._client.init_multipart_upload(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
-    def put(self, key, data) -> models.PutObjectResult:
-        """文件上传"""
-        return self._client.put_object(key, data)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
-    def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
-        """分片上传"""
-        return self._client.upload_part(key, upload_id, position, data)
-
-    def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
-        """获取分片列表"""
-        return [part_info for part_info in PartIterator(self._client, key, upload_id)]
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
-    def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
-        """合并分片"""
-        headers = None if parts else {"x-oss-complete-all": "yes"}
-        return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
-    def append(self, key, position, data) -> models.AppendObjectResult:
-        """追加上传"""
-        return self._client.append_object(key, position, data)
-
-    def iter_data(self, data, chunk_size=None):
-        chunk_size = chunk_size or self.chunk_size
-        if isinstance(data, Response):
-            for part_data in data.iter_content(chunk_size):
-                yield part_data
-        if isinstance(data, bytes):
-            for i in range(0, len(data), chunk_size):
-                yield data[i:i + chunk_size]
-
-    def assemble(self, ready_data, data, chunk_size=None):
-        upload_data = b""
-        ready_data = ready_data + data
-        chunk_size = chunk_size or self.chunk_size
-        if len(ready_data) >= chunk_size:
-            upload_data = ready_data[:chunk_size]
-            ready_data = ready_data[chunk_size:]
-        return ready_data, upload_data
-
-    def content_length(self, key: str) -> int:
-        head = self.head(key)
-        return head.content_length
-