cobweb-launcher 0.1.24__py3-none-any.whl → 1.0.0__py3-none-any.whl

cobweb/launchers/launcher.py

@@ -0,0 +1,199 @@
+import time
+
+import setting
+import inspect
+import threading
+import importlib
+
+from cobweb.base import Seed, Queue
+from cobweb.utils.tools import dynamic_load_class
+
+
+class Launcher(threading.Thread):
+
+    SEEDS = []
+
+    __DOING__ = {}
+
+    __CUSTOM_FUNC__ = {
+        "download": None,
+        "download_midware": None,
+        "parse": None,
+    }
+
+    __LAUNCHER_QUEUE__ = {
+        "new": Queue(),
+        "todo": Queue(),
+        "done": Queue(),
+    }
+
+    __LAUNCHER_FUNC__ = [
+        "_reset",
+        "_scheduler",
+        "_insert",
+        "_refresh",
+        "_delete",
+    ]
+
+    def __init__(self, task, project, custom_setting=None):
+        super().__init__()
+        self.task = task
+        self.project = project
+
+        self._stop = threading.Event()  # 结束事件
+        self._pause = threading.Event()  # 暂停事件
+
+        if custom_setting:
+            setting_ = dict()
+            if isinstance(custom_setting, dict):
+                setting_ = custom_setting
+            else:
+                if isinstance(custom_setting, str):
+                    custom_setting = importlib.import_module(custom_setting)
+                if not inspect.ismodule(custom_setting):
+                    raise Exception
+                for k, v in custom_setting.__dict__.items():
+                    if not k.startswith("__") and not inspect.ismodule(v):
+                        setting_[k] = v
+            for k, v in setting_.items():
+                setattr(setting, k, v)
+
+        self._Crawler = dynamic_load_class(setting.CRAWLER)
+        self._Pipeline = dynamic_load_class(setting.PIPELINE)
+
+        self._scheduler_wait_seconds = setting.SCHEDULER_WAIT_SECONDS
+        self._todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
+        self._new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
+        self._done_queue_wait_seconds = setting.DONE_QUEUE_WAIT_SECONDS
+        self._upload_queue_wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
+        self._seed_reset_seconds = setting.SEED_RESET_SECONDS
+
+        self._todo_queue_size = setting.TODO_QUEUE_SIZE
+        self._new_queue_max_size = setting.NEW_QUEUE_MAX_SIZE
+        self._done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
+        self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
+
+        self._done_model = setting.DONE_MODEL
+
+        self._upload_queue = Queue()
+
+    @property
+    def start_seeds(self):
+        return [Seed(seed) for seed in self.SEEDS]
+
+    @property
+    def request(self):
+        """
+        自定义request函数
+        use case:
+            from cobweb.base import Request, BaseItem
+            @launcher.request
+            def request(seed: Seed) -> Union[Request, BaseItem]:
+                ...
+                return Request(seed.url, seed)
+        """
+        def decorator(func):
+            self.__CUSTOM_FUNC__["request"] = func
+        return decorator
+
+    @property
+    def download(self):
+        """
+        自定义download函数
+        use case:
+            from cobweb.base import Request, Response, Seed, BaseItem
+            @launcher.download
+            def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+                ...
+                yield Response(item.seed, response)
+        """
+        def decorator(func):
+            self.__CUSTOM_FUNC__["download"] = func
+        return decorator
+
+    @property
+    def parse(self):
+        """
+        自定义parse函数, xxxItem为自定义的存储数据类型
+        use case:
+            from cobweb.base import Request, Response
+            @launcher.download
+            def download(item: Response) -> BaseItem:
+               ...
+               yield xxxItem(seed, **kwargs)
+        """
+        def decorator(func):
+            self.__CUSTOM_FUNC__["parse"] = func
+        return decorator
+
+    def _remove_doing_seeds(self, seeds):
+        for seed in seeds:
+            self.__DOING__.pop(seed, None)
+
+    def _execute_heartbeat(self):
+        pass
+
+    def _reset(self):
+        pass
+
+    def _scheduler(self):
+        pass
+
+    def _insert(self):
+        pass
+
+    def _refresh(self):
+        pass
+
+    def _delete(self):
+        pass
+
+    def _execute(self):
+        for func_name in self.__LAUNCHER_FUNC__:
+            threading.Thread(name=func_name, target=getattr(self, func_name)).start()
+            time.sleep(2)
+
+    def _polling(self):
+
+        check_emtpy_times = 0
+
+        while not self._stop.is_set():
+
+            queue_not_empty_count = 0
+
+            for q in self.__LAUNCHER_QUEUE__.values():
+                if q.length != 0:
+                    queue_not_empty_count += 1
+
+            if self._pause.is_set() and queue_not_empty_count != 0:
+                self._pause.clear()
+                self._execute()
+
+            elif queue_not_empty_count == 0:
+                check_emtpy_times += 1
+            else:
+                check_emtpy_times = 0
+
+            if check_emtpy_times > 2:
+                check_emtpy_times = 0
+                self.__DOING__ = {}
+                self._pause.set()
+
+    def run(self):
+        threading.Thread(target=self._execute_heartbeat).start()
+
+        self._Crawler(
+            upload_queue=self._upload_queue,
+            custom_func=self.__CUSTOM_FUNC__,
+            launcher_queue=self.__LAUNCHER_QUEUE__,
+        ).start()
+
+        self._Pipeline(
+            upload_queue=self._upload_queue,
+            done_queue=self.__LAUNCHER_QUEUE__["done"],
+            upload_queue_size=self._upload_queue_max_size,
+            upload_wait_seconds=self._upload_queue_wait_seconds
+        ).start()
+
+        self._execute()
+        self._polling()

cobweb/launchers/launcher_pro.py

@@ -0,0 +1,174 @@
+import time
+import threading
+
+from cobweb.db import RedisDB
+from cobweb.base import Seed, logger
+from cobweb.launchers import Launcher
+from cobweb.constant import DealModel, LogTemplate
+
+
+class LauncherPro(Launcher):
+
+    def __init__(self, task, project, custom_setting=None):
+        super().__init__(task, project, custom_setting)
+        self._todo = "{%s:%s}:todo" % (project, task)
+        self._done = "{%s:%s}:done" % (project, task)
+        self._fail = "{%s:%s}:fail" % (project, task)
+        self._heartbeat = "heartbeat:%s_%s" % (project, task)
+        self._reset_lock = "lock:reset:%s_%s" % (project, task)
+        self._heartbeat_lock = "lock:heartbeat:%s_%s" % (project, task)
+        self._db = RedisDB()
+
+        self._heartbeat_start_event = threading.Event()
+        self._redis_queue_empty_event = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat)
+
+    def _execute_heartbeat(self):
+        while not self._stop.is_set():
+            if self._heartbeat_start_event.is_set():
+                if self._db.lock(self._heartbeat_lock, t=1):
+                    self._db.setex(self._heartbeat, 3)
+                time.sleep(1)
+            time.sleep(0.5)
+
+    def _reset(self):
+        """
+        检查过期种子，重新添加到redis缓存中
+        """
+        first = True
+        while not self._pause.is_set():
+            reset_wait_seconds = 15
+            if self._db.lock(self._reset_lock, t=120):
+                if not self.heartbeat:
+                    self._heartbeat_start_event.set()
+
+                _min = -int(time.time()) + self._seed_reset_seconds \
+                    if self.heartbeat or not first else "-inf"
+
+                self._db.members(
+                    self._todo, 0,
+                    _min=_min, _max="(0"
+                )
+                self._db.delete(self._reset_lock)
+                reset_wait_seconds = 60
+
+            time.sleep(reset_wait_seconds)
+            first = False
+
+    def _scheduler(self):
+        """
+        调度任务，获取redis队列种子，同时添加到doing字典中
+        """
+        if self.start_seeds:
+            self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
+        while not self._pause.is_set():
+            if not self._db.zcount(self._todo, 0, "(1000"):
+                time.sleep(self._scheduler_wait_seconds)
+                continue
+            if self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
+                time.sleep(self._todo_queue_full_wait_seconds)
+                continue
+            members = self._db.members(
+                self._todo, int(time.time()),
+                count=self._todo_queue_size,
+                _min=0, _max="(1000"
+            )
+            for member, priority in members:
+                seed = Seed(member, priority=priority)
+                self.__LAUNCHER_QUEUE__['todo'].push(seed)
+                self.__DOING__[seed.to_string] = seed.params.priority
+
+    def _insert(self):
+        """
+        添加新种子到redis队列中
+        """
+        while not self._pause.is_set():
+            seeds = {}
+            status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+            for _ in range(self._new_queue_max_size):
+                seed = self.__LAUNCHER_QUEUE__['new'].pop()
+                if not seed:
+                    break
+                seeds[seed.to_string] = seed.params.priority
+            if seeds:
+                self._db.zadd(self._todo, seeds, nx=True)
+            if status:
+                time.sleep(self._new_queue_wait_seconds)
+
+    def _refresh(self):
+        """
+        刷新doing种子过期时间，防止reset重新消费
+        """
+        while not self._pause.is_set():
+            if self.__DOING__:
+                refresh_time = int(time.time())
+                seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
+                self._db.zadd(self._todo, item=seeds, xx=True)
+            time.sleep(30)
+
+    def _delete(self):
+        """
+        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
+        """
+        while not self._pause.is_set():
+            seeds, s_seeds, f_seeds = [], [], []
+            status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+
+            for _ in range(self._done_queue_max_size):
+                seed = self.__LAUNCHER_QUEUE__['done'].pop()
+                if not seed:
+                    break
+                if seed.params.identifier == DealModel.fail:
+                    f_seeds.append(seed.to_string)
+                elif self._done_model == 1:
+                    s_seeds.append(seed.to_string)
+                else:
+                    seeds.append(seed.to_string)
+            if seeds:
+                self._db.zrem(self._todo, *seeds)
+            if s_seeds:
+                self._db.done([self._todo, self._done], *s_seeds)
+            if f_seeds:
+                self._db.done([self._todo, self._fail], *f_seeds)
+
+            self._remove_doing_seeds(seeds)
+
+            if status:
+                time.sleep(self._done_queue_wait_seconds)
+
+    def _polling(self):
+        check_emtpy_times = 0
+        while not self._stop.is_set():
+            queue_not_empty_count = 0
+            pooling_wait_seconds = 30
+            if not self._db.zcard(self._todo):
+                for q in self.__LAUNCHER_QUEUE__.values():
+                    if q.length != 0:
+                        queue_not_empty_count += 1
+                if self._pause.is_set() and queue_not_empty_count != 0:
+                    self._pause.clear()
+                    self._execute()
+                elif queue_not_empty_count == 0:
+                    pooling_wait_seconds = 3
+                    check_emtpy_times += 1
+                else:
+                    check_emtpy_times = 0
+                if check_emtpy_times > 2:
+                    check_emtpy_times = 0
+                    self.__DOING__ = {}
+                    self._pause.set()
+            if not self._pause.is_set():
+                logger.info(LogTemplate.launcher_pro_polling.format(
+                    task=self.task,
+                    doing_len=len(self.__DOING__.keys()),
+                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
+                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
+                    redis_seed_count=self._db.zcount(self._todo, "-inf", "+inf"),
+                    redis_todo_len=self._db.zcount(self._todo, 0, "(1000"),
+                    redis_doing_len=self._db.zcount(self._todo, "-inf", "(0"),
+                    upload_len=self._upload_queue.length
+                ))
+            time.sleep(pooling_wait_seconds)

cobweb/pipelines/__init__.py

@@ -0,0 +1,2 @@
+from .base_pipeline import Pipeline
+from .loghub_pipeline import LoghubPipeline

cobweb/pipelines/base_pipeline.py

@@ -0,0 +1,54 @@
+import time
+import threading
+
+from abc import ABC, abstractmethod
+from cobweb.base import BaseItem, Queue, logger
+
+
+class Pipeline(threading.Thread, ABC):
+
+    def __init__(
+            self,
+            done_queue: Queue,
+            upload_queue: Queue,
+            upload_queue_size: int,
+            upload_wait_seconds: int
+    ):
+        super().__init__()
+        self.done_queue = done_queue
+        self.upload_queue = upload_queue
+        self.upload_queue_size = upload_queue_size
+        self.upload_wait_seconds = upload_wait_seconds
+
+    @abstractmethod
+    def build(self, item: BaseItem) -> dict:
+        pass
+
+    @abstractmethod
+    def upload(self, table: str, data: list) -> bool:
+        pass
+
+    def run(self):
+        while True:
+            status = self.upload_queue.length < self.upload_queue_size
+            if status:
+                time.sleep(self.upload_wait_seconds)
+            data_info, seeds = {}, []
+            for _ in range(self.upload_queue_size):
+                item = self.upload_queue.pop()
+                if not item:
+                    break
+                data = self.build(item)
+                seeds.append(item.seed)
+                data_info.setdefault(item.table, []).append(data)
+            for table, datas in data_info.items():
+                try:
+                    self.upload(table, datas)
+                    status = True
+                except Exception as e:
+                    logger.info(e)
+                    status = False
+                if status:
+                    self.done_queue.push(seeds)
+
+

cobweb/pipelines/loghub_pipeline.py

@@ -0,0 +1,34 @@
+import json
+import setting
+
+from cobweb.base import BaseItem
+from cobweb.pipelines import Pipeline
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+
+class LoghubPipeline(Pipeline):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = LogClient(**setting.LOGHUB_CONFIG)
+
+    def build(self, item: BaseItem):
+        log_item = LogItem()
+        temp = item.to_dict
+        for key, value in temp.items():
+            if not isinstance(value, str):
+                temp[key] = json.dumps(value, ensure_ascii=False)
+        contents = sorted(temp.items())
+        log_item.set_contents(contents)
+        return log_item
+
+    def upload(self, table, datas):
+        request = PutLogsRequest(
+            project=setting.LOGHUB_PROJECT,
+            logstore=table,
+            topic=setting.LOGHUB_TOPIC,
+            source=setting.LOGHUB_SOURCE,
+            logitems=datas,
+            compress=True
+        )
+        self.client.put_logs(request=request)

cobweb/setting.py

@@ -1,13 +1,62 @@
 import os
 
+# redis db config
+REDIS_CONFIG = {
+    "host": os.getenv("REDIS_HOST"),
+    "password": os.getenv("REDIS_PASSWORD"),
+    "port": int(os.getenv("REDIS_PORT", 6379)),
+    "db": int(os.getenv("REDIS_DB", 0)),
+}
 
-# model: 0, 1, 2
-MODEL = int(os.getenv("MODEL", "0"))
+# loghub db config
+LOGHUB_TOPIC = os.getenv("LOGHUB_TOPIC")
+LOGHUB_SOURCE = os.getenv("LOGHUB_SOURCE")
+LOGHUB_PROJECT = os.getenv("LOGHUB_PROJECT")
+LOGHUB_CONFIG = {
+    "endpoint": os.getenv("LOGHUB_ENDPOINT"),
+    "accessKeyId": os.getenv("LOGHUB_ACCESS_KEY"),
+    "accessKey": os.getenv("LOGHUB_SECRET_KEY")
+}
 
-# 重制score值的等待时间, 默认10分钟
-RESET_SCORE = int(os.getenv("RESET_SCORE", "600"))
+# oss util config
+OSS_BUCKET = os.getenv("OSS_BUCKET")
+OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
+OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
+OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
+OSS_MIN_UPLOAD_SIZE = 1024 * 100
+OSS_CHUNK_SIZE = 1024 ** 2
 
-# 默认设置检查spider queue队列锁的存活时间为30s
-CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
+# 采集器选择
+CRAWLER = "cobweb.crawlers.CrawlerAir"
 
+# 数据上传链路
+PIPELINE = "cobweb.pipelines.loghub_pipeline.LoghubPipeline"
 
+
+# Launcher 等待时间
+SCHEDULER_WAIT_SECONDS = 15  # 调度等待时间
+TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
+NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
+DONE_QUEUE_WAIT_SECONDS = 15   # done队列等待时间
+UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
+SEED_RESET_SECONDS = 600   # 种子重制时间
+
+
+# Launcher 队列长度
+TODO_QUEUE_SIZE = 100  # todo队列长度
+NEW_QUEUE_MAX_SIZE = 100  # new队列长度
+DONE_QUEUE_MAX_SIZE = 100  # done队列长度
+UPLOAD_QUEUE_MAX_SIZE = 100  # upload队列长度
+
+# DONE_MODEL IN (0, 1), 种子完成模式
+DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加至失败队列；1:种子消费成功添加至成功队列，失败添加至失败队列
+
+# DOWNLOAD_MODEL IN (0, 1), 下载模式
+DOWNLOAD_MODEL = 0  # 0: 通用下载；1:文件下载
+
+# spider
+SPIDER_THREAD_NUM = 10
+SPIDER_MAX_RETRIES = 5
+
+# 文件下载响应类型过滤
+FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]

cobweb/utils/__init__.py

@@ -0,0 +1,3 @@
+from .oss import OssUtil
+from .tools import *
+

cobweb/utils/oss.py

@@ -0,0 +1,87 @@
+import setting
+
+from requests import Response
+from oss2 import Auth, Bucket, models
+from cobweb.exceptions import oss_db_exception
+from base.decorators import decorator_oss_db
+
+
+class OssUtil:
+
+    def __init__(
+        self,
+        bucket=None,
+        endpoint=None,
+        access_key=None,
+        secret_key=None,
+        chunk_size=None,
+        min_upload_size=None,
+    ):
+        self.bucket = bucket or setting.OSS_BUCKET
+        self.endpoint = endpoint or setting.OSS_ENDPOINT
+        self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
+        self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
+        self._auth = Auth(
+            access_key_id=access_key or setting.OSS_ACCESS_KEY,
+            access_key_secret=secret_key or setting.OSS_SECRET_KEY
+        )
+        self._client = Bucket(
+            auth=self._auth,
+            endpoint=self.endpoint,
+            bucket_name=self.bucket,
+        )
+
+    def exists(self, key: str) -> bool:
+        return self._client.object_exists(key)
+
+    def head(self, key: str) -> models.HeadObjectResult:
+        return self._client.head_object(key)
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
+    def init_part(self, key) -> models.InitMultipartUploadResult:
+        """初始化分片上传"""
+        return self._client.init_multipart_upload(key)
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
+    def put(self, key, data) -> models.PutObjectResult:
+        """文件上传"""
+        return self._client.put_object(key, data)
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
+    def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
+        """分片上传"""
+        return self._client.upload_part(key, upload_id, position, data)
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
+    def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
+        """合并分片"""
+        headers = None if parts else {"x-oss-complete-all": "yes"}
+        return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
+    def append(self, key, position, data) -> models.AppendObjectResult:
+        """追加上传"""
+        return self._client.append_object(key, position, data)
+
+    def iter_data(self, data, chunk_size=None):
+        chunk_size = chunk_size or self.chunk_size
+        if isinstance(data, Response):
+            for part_data in data.iter_content(chunk_size):
+                yield part_data
+        if isinstance(data, bytes):
+            for i in range(0, len(data), chunk_size):
+                yield data[i:i + chunk_size]
+
+    def assemble(self, ready_data, data, chunk_size=None):
+        upload_data = b""
+        ready_data = ready_data + data
+        chunk_size = chunk_size or self.chunk_size
+        if len(ready_data) >= chunk_size:
+            upload_data = ready_data[:chunk_size]
+            ready_data = ready_data[chunk_size:]
+        return ready_data, upload_data
+
+    def content_length(self, key: str) -> int:
+        head = self.head(key)
+        return head.content_length
+

cobweb/utils/tools.py

@@ -0,0 +1,42 @@
+import re
+import hashlib
+from typing import Union
+from importlib import import_module
+
+
+def md5(text: Union[str, bytes]) -> str:
+    if isinstance(text, str):
+        text = text.encode('utf-8')
+    return hashlib.md5(text).hexdigest()
+
+
+def build_path(site, url, file_type):
+    return f"{site}/{md5(url)}.{file_type}"
+
+
+def format_size(content_length: int) -> str:
+    units = ["KB", "MB", "GB", "TB"]
+    for i in range(4):
+        num = content_length / (1024 ** (i + 1))
+        if num < 1024:
+            return f"{round(num, 2)} {units[i]}"
+
+
+def dynamic_load_class(model_info):
+    if isinstance(model_info, str):
+        if "import" in model_info:
+            model_path, class_name = re.search(
+                r"from (.*?) import (.*?)$", model_info
+            ).groups()
+            model = import_module(model_path)
+            class_object = getattr(model, class_name)
+        else:
+            model_path, class_name = model_info.rsplit(".", 1)
+            model = import_module(model_path)
+            class_object = getattr(model, class_name)
+        return class_object
+    raise TypeError()
+
+
+def download_log_info(item:dict) -> str:
+    return "\n".join([" " * 12 + f"{k.ljust(14)}:    {v}" for k, v in item.items()])

{cobweb_launcher-0.1.24.dist-info → cobweb_launcher-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.24
+Version: 1.0.0
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP