cobweb-launcher 0.1.7__py3-none-any.whl → 1.2.41__py3-none-any.whl

cobweb/db/oss_db.py

@@ -1,127 +0,0 @@
-import oss2
-from typing import Union
-from oss2.models import PartInfo
-from requests import Response
-from cobweb import log
-
-
-class OssDB:
-
-    def __init__(
-            self,
-            bucket_name,
-            endpoint,
-            access_key,
-            secret_key,
-            chunk_size,
-            min_size
-    ):
-        self.endpoint = endpoint
-        self.bucket_name = bucket_name
-        self.auth = oss2.Auth(
-            access_key_id=access_key,
-            access_key_secret=secret_key
-        )
-        self.bucket = oss2.Bucket(
-            auth=self.auth,
-            endpoint=endpoint,
-            bucket_name=bucket_name
-        )
-        self.chunk_size = chunk_size or 1024 ** 2
-        self.min_size = min_size or 1024
-
-    @staticmethod
-    def format_upload_len(length):
-        if not length:
-            raise ValueError("Length cannot be None or 0")
-
-        units = ["KB", "MB", "GB", "TB"]
-        for i in range(3):
-            num = length / (1024 ** (i + 1))
-            if num <= 1024:
-                return f"{round(num, 2)} {units[i]}"
-
-    def assemble(self, ready_data, part_data):
-        upload_data = None
-        ready_data = ready_data + part_data
-        if len(ready_data) >= self.chunk_size:
-            upload_data = ready_data[:self.chunk_size]
-            ready_data = ready_data[self.chunk_size:]
-
-        return ready_data, upload_data
-
-    def iter_data(self, data):
-        if isinstance(data, Response):
-            for part_data in data.iter_content(self.chunk_size):
-                yield part_data
-        if isinstance(data, bytes):
-            for i in range(0, len(data), self.chunk_size):
-                yield data[i:i + self.chunk_size]
-
-    def upload_split(
-            self, oss_path: str,
-            data: Union[bytes, Response],
-            timeout: int = 300,
-    ):
-        parts = []
-        status = False
-        upload_id = None
-        ready_data = b""
-        upload_data_len = 0
-        headers = {"Expires": str(timeout * 1000)}
-        try:
-            upload_id = self.bucket.init_multipart_upload(oss_path).upload_id
-            for part_data in self.iter_data(data):
-                upload_data_len += len(part_data)
-                ready_data, upload_data = self.assemble(ready_data, part_data)
-                if upload_data:
-                    part_index = len(parts) + 1
-                    upload_info = self.bucket.upload_part(
-                        oss_path, upload_id, part_index, upload_data
-                    )
-                    parts.append(PartInfo(part_index, upload_info.etag))
-
-            format_upload = self.format_upload_len(upload_data_len)
-
-            if parts and ready_data:
-                part_index = len(parts) + 1
-                upload_info = self.bucket.upload_part(
-                    oss_path, upload_id, part_index, ready_data
-                )
-                parts.append(PartInfo(part_index, upload_info.etag))
-                self.bucket.complete_multipart_upload(
-                    oss_path, upload_id, parts
-                )
-                log.info(
-                    f"split upload, file path: {oss_path}"
-                    f", file size: {format_upload}"
-                )
-
-            elif len(ready_data) > self.min_size:
-                self.bucket.put_object(oss_path, ready_data, headers)
-                log.info(
-                    f"upload file, file path: {oss_path}"
-                    f", file size: {format_upload}"
-                )
-
-            else:
-                log.info(
-                    f"file size smaller than min size! "
-                    f"file size: {format_upload}"
-                )
-            status = True
-        except ValueError as e:
-            log.exception(str(e))
-        except oss2.exceptions.RequestError as e:
-            self.bucket = oss2.Bucket(
-                auth=self.auth,
-                endpoint=self.endpoint,
-                bucket_name=self.bucket_name
-            )
-            log.exception("oss timeout! " + str(e))
-        except Exception as e:
-            self.bucket.abort_multipart_upload(oss_path, upload_id, headers)
-            log.exception("upload file exception: " + str(e))
-
-        return status
-

cobweb/db/scheduler/default.py

@@ -1,8 +0,0 @@
-from cobweb import SchedulerInterface
-
-
-class Default(SchedulerInterface):
-
-    def schedule(self):
-        pass
-

cobweb/db/scheduler/textfile.py

@@ -1,27 +0,0 @@
-from cobweb import log, Seed, SchedulerInterface
-
-
-class Textfile(SchedulerInterface):
-
-    index = None
-
-    def schedule(self):
-        try:
-            seeds = []
-            with open(self.table, "r") as fp:
-                fp.seek(self.index or 0, 0)
-                for _ in range(self.length):
-                    data = fp.readline().strip()
-                    if not data:
-                        log.info("scheduler end!")
-                        self.stop = True
-                        break
-                    seeds.append(Seed(data))
-                    self.index = fp.tell()
-            return seeds
-        except FileNotFoundError:
-            log.error("task table not found!")
-            return None
-        except TypeError:
-            log.error("task table type error!")
-            return None

cobweb/db/storer/console.py

@@ -1,9 +0,0 @@
-from cobweb import log, StorerInterface
-
-
-class Console(StorerInterface):
-
-    def store(self, data_list):
-        for item in data_list:
-            log.info(f"item info: {item}")
-

cobweb/db/storer/loghub.py

@@ -1,54 +0,0 @@
-import json
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-from cobweb import log, StorerInterface
-
-
-class Loghub(StorerInterface):
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.client = None
-
-    def init_loghub_clint(self):
-        try:
-            self.client = LogClient(
-                self.config['endpoint'],
-                self.config['access_key_id'],
-                self.config['access_key']
-            )
-        except Exception as e:
-            self.client = None
-            return False
-
-    def store(self, data_list):
-        try:
-            if not self.client:
-                self.init_loghub_clint()
-
-            log_items = list()
-            for item in data_list:
-                temp = item._asdict()
-                for key, value in temp.items():
-                    if isinstance(value, str):
-                        temp[key] = value
-                    else:
-                        temp[key] = json.dumps(value, ensure_ascii=False)
-                log_item = LogItem()
-                contents = sorted(temp.items())  # dict to tuple
-                log_item.set_contents(contents)
-                log_items.append(log_item)
-            request = PutLogsRequest(
-                project=self.config["project"],
-                logstore=self.table,
-                topic=self.config["topic"],
-                source=self.config.get("source"),
-                logitems=log_items,
-                compress=True
-            )
-            self.client.put_logs(request=request)
-            log.info(f"save data, data length: {len(data_list)}")
-            return True
-        except Exception as e:
-            log.exception(e)
-            return False
-

cobweb/db/storer/redis.py

@@ -1,15 +0,0 @@
-from cobweb import log, StorerInterface
-
-
-class Redis(StorerInterface):
-
-    def store(self, data_list):
-        try:
-            data_str = "\n".join(str(data) for data in data_list)
-            with open(self.table, "a") as fp:
-                fp.write(data_str)
-            log.info(f"save data, data length: {len(data_list)}")
-            return True
-        except Exception as e:
-            return False
-

cobweb/db/storer/textfile.py

@@ -1,15 +0,0 @@
-from cobweb import log, StorerInterface
-
-
-class Textfile(StorerInterface):
-
-    def store(self, data_list):
-        try:
-            data_str = "\n".join(str(data) for data in data_list)
-            with open(self.table, "a") as fp:
-                fp.write(data_str)
-            log.info(f"save data, data length: {len(data_list)}")
-            return True
-        except Exception as e:
-            return False
-

cobweb/decorators.py

@@ -1,16 +0,0 @@
-from functools import wraps
-from cobweb import log
-
-
-def check_redis_status(func):
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            result = func(*args, **kwargs)
-        except Exception as e:
-            log.exception(e)
-            result = False
-        return result
-
-    return wrapper
-

cobweb/distributed/launcher.py

@@ -1,243 +0,0 @@
-import time
-import threading
-from threading import Thread
-
-from .models import Scheduler, Spider, Storer
-from cobweb import log, Queue, DBItem, RedisDB
-from cobweb.setting import MODEL, RESET_SCORE, CHECK_LOCK_TIME
-from cobweb.utils import (
-    struct_queue_name as sqn,
-    restore_table_name as rtn,
-    parse_import_model as pim,
-)
-
-
-def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
-    log.info("run check thread after 30 seconds...")
-    time.sleep(30)
-    spider_info = """
-------------------- check: {0} ------------------
-            running_spider_thread_num: {1}
-            redis_ready_seed_length:   {2}
-            redis_spider_seed_length:  {3}
-            memory_seed_queue_length:  {4}
-            storer_upload_queue_length_info: 
-                {5}
------------------------  end  -----------------------"""
-    while True:
-        status = "running"
-        running_spider_thread_num = spider.spider_in_progress.length
-        redis_ready_seed_length = ready_seed_length()
-        redis_spider_seed_length = spider_queue_length()
-        memory_seed_queue_length = scheduler.queue.length
-        storer_upload_queue_list = []
-        for storer in storer_list:
-            storer_upload_queue_list.append(
-                f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
-            )
-        if (
-                scheduler.stop and
-                # not redis_ready_seed_length and
-                not memory_seed_queue_length and
-                not running_spider_thread_num
-        ):
-            if not MODEL:
-                log.info("spider is done?")
-            last.set()
-            time.sleep(3)
-            storer_queue_empty = True
-            storer_upload_queue_list = []
-            for storer in storer_list:
-                if storer.queue.length:
-                    storer_queue_empty = False
-                storer_upload_queue_list.append(
-                    f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
-                )
-            if (
-                    storer_queue_empty and
-                    not redis_ready_seed_length and
-                    not redis_spider_seed_length
-            ):
-                if MODEL:
-                    log.info("waiting for push seeds...")
-                    status = "waiting"
-                    time.sleep(30)
-                else:
-                    log.info("spider done!")
-                    break
-
-            last.clear()
-
-        storer_upload_queue_length_info = "\n            ".join(
-            storer_upload_queue_list) if storer_upload_queue_list else "None"
-        log.info(spider_info.format(
-            status,
-            running_spider_thread_num,
-            redis_ready_seed_length,
-            redis_spider_seed_length,
-            memory_seed_queue_length,
-            storer_upload_queue_length_info
-        ))
-
-        time.sleep(3)
-    stop.set()
-
-
-def launcher(task):
-    """
-    任务启动装饰器
-    :param task: 任务配置信息
-    """
-    def decorator(func):
-        """
-        Item:
-            Textfile()
-            Loghub()
-            Console()
-        e.g.
-        task.fields = "a,b"
-        func(item, seed)
-            a = "a"
-            b = "b"
-            data = {"a": "a", "b": "b"}
-            yield item.Loghub(**data)
-            yield item.Loghub(a=a, b=b)
-        """
-        storer_list = []
-
-        # 程序结束事件
-        last = threading.Event()
-        # 停止采集事件
-        stop = threading.Event()
-
-        # 初始化redis信息
-        redis_db = RedisDB(
-            task.project, task.task_name, task.redis_info,
-            model=MODEL, cs_lct=CHECK_LOCK_TIME, rs_time=RESET_SCORE
-        )
-
-        log.info("初始化cobweb!")
-
-        seed_queue = Queue()
-
-        if task.scheduler_info is None:
-            task.scheduler_info = dict()
-
-        # 调度器动态继承
-        sql = task.scheduler_info.get("sql")
-        table = task.scheduler_info.get("table")
-        size = task.scheduler_info.get("size")
-        scheduler_config = task.scheduler_info.get("config")
-        scheduler_db = task.scheduler_info.get("db", "default")
-        DB, class_name = pim(scheduler_db, "scheduler")
-        # SchedulerDB, table, sql, length, size, config = task.scheduler_info
-        SchedulerTmp = type(class_name, (Scheduler, DB), {})
-
-        # 初始化调度器
-        scheduler = SchedulerTmp(
-            table=table, sql=sql, size=size, queue=seed_queue,
-            length=task.scheduler_queue_length, config=scheduler_config
-        )
-
-        # 初始化采集器
-        spider = Spider(seed_queue, task.max_retries)
-
-        # 解析存储器信息
-        storer_info_list = task.storer_info or []
-        if not isinstance(storer_info_list, list):
-            storer_info_list = [storer_info_list]
-
-        # new item
-        item = type("Item", (object,), {"redis_client": redis_db.client})()
-
-        for storer_info in storer_info_list:
-            storer_db = storer_info["db"]
-            fields = storer_info["fields"]
-            storer_table = storer_info.get("table", "console")
-            storer_config = storer_info.get("config")
-
-            StorerDB, class_name = pim(storer_db, "storer")
-            StorerTmp = type(class_name, (Storer, StorerDB), {})
-
-            db_name = class_name.lower()
-            if not getattr(item, db_name, None):
-                instance = type(db_name, (DBItem,), {})
-                setattr(item, db_name, instance)
-
-            storer_item_instance = getattr(item, db_name)
-            storer_item_instance.init_item(storer_table, fields)
-
-            storer_queue = sqn(db_name, storer_table)
-            queue = getattr(storer_item_instance, storer_queue)
-            # 初始话存储器
-            table_name = rtn(table_name=storer_table)
-            storer = StorerTmp(
-                table=table_name, fields=fields,
-                length=task.storer_queue_length,
-                queue=queue, config=storer_config
-            )
-            storer_list.append(storer)
-
-        Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
-        Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
-
-        # 推送初始种子
-        # seeds = start_seeds(task.start_seed)
-        redis_db.add_seed(task.seeds)
-        # 启动调度器, 调度至redis队列
-        Thread(
-            # name="xxxx_schedule_seeds",
-            target=scheduler.schedule_seed,
-            args=(
-                redis_db.ready_seed_length,
-                redis_db.get_scheduler_lock,
-                redis_db.add_seed
-            )
-        ).start()
-
-        # 启动调度器, 调度任务队列
-        Thread(
-            # name="xxxx_schedule_task",
-            target=scheduler.schedule_task,
-            args=(
-                stop, redis_db.get_seed,
-                redis_db.ready_seed_length
-            )
-        ).start()
-
-        # 启动采集器
-        for index in range(task.spider_num):
-            Thread(
-                # name=f"xxxx_spider_task:{index}",
-                target=spider.spider_task,
-                args=(
-                    stop, func, item,
-                    redis_db.del_seed
-                )
-            ).start()
-
-        # 启动存储器
-        for storer in storer_list:
-            Thread(
-                # name=f"xxxx_store_task:{storer.table}",
-                target=storer.store_task,
-                args=(
-                    stop, last,
-                    redis_db.reset_seed,
-                    redis_db.set_storer
-                )
-            ).start()
-
-        Thread(
-            # name="check_spider",
-            target=check,
-            args=(
-                stop, last, spider,
-                scheduler, storer_list,
-                redis_db.ready_seed_length,
-                redis_db.spider_queue_length,
-            )
-        ).start()
-
-    return decorator
-

cobweb/distributed/models.py

@@ -1,143 +0,0 @@
-import time
-from hashlib import md5
-from cobweb import log, Queue, Seed
-from cobweb.utils import issubclass_cobweb_inf
-
-# from pympler import asizeof
-
-
-class Scheduler:
-
-    def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
-
-        inf_name = "SchedulerInterface"
-        if not issubclass_cobweb_inf(self.__class__, inf_name):
-            raise Exception("not have schedule function!")
-
-        if self.__class__.__name__ == "Default":
-            self.stop = True
-            return None
-
-        while not self.stop:
-            length = ready_seed_length()
-            if length > self.size:
-                time.sleep(15)
-
-            elif get_scheduler_lock():
-                seeds = self.schedule()
-                add_seed(seeds)
-
-        log.info(f"close thread: schedule_seed")
-
-    def schedule_task(self, stop, get_seed, ready_seed_length):
-        time.sleep(3)
-        while not stop.is_set():
-
-            if not ready_seed_length():
-                time.sleep(15)
-                continue
-
-            if self.queue.length >= self.length:
-                time.sleep(3)
-                continue
-
-            seeds = get_seed(self.length)
-            self.queue.push(seeds)
-        log.info(f"close thread: schedule_task")
-
-
-class Spider:
-
-    def __init__(self, queue, max_retries=5):
-        self.spider_in_progress = Queue()
-        self.max_retries = max_retries
-        self.queue = queue
-
-    def spider_task(self, stop, func, item, del_seed):
-        while not stop.is_set():
-            seed = self.queue.pop()
-            if not seed:
-                time.sleep(3)
-                continue
-            elif seed._retry >= self.max_retries:
-                del_seed(seed, spider_status=False)
-                continue
-            try:
-                self.spider_in_progress.push(1, direct_insertion=True)
-                # log.info("spider seed: " + str(seed))
-                ret_count = 0
-                status = None
-                store_queue = None
-                store_data = list()
-                for it in func(item, seed):
-                    ret_count += 1
-                    if getattr(it, "table_name", None):
-                        if not store_queue:
-                            store_queue = it.queue()
-                        store_data.append(it.struct_data)
-                    elif isinstance(it, Seed):
-                        self.queue.push(it)
-                    elif any(isinstance(it, t) for t in (list, tuple)):
-                        self.queue.push([s if isinstance(s, Seed) else Seed(s) for s in it])
-                    elif isinstance(it, bool):
-                        status = it
-
-                if store_queue and store_data:
-                    store_data.append(seed)
-                    store_queue.push(store_data)
-
-                if status:
-                    del_seed(seed, spider_status=True)
-                elif not ret_count or status is False:
-                    seed._retry += 1
-                    self.queue.push(seed)
-
-            except Exception as e:
-                seed._retry += 1
-                self.queue.push(seed)
-                log.info(f"{str(seed)} -> {str(e)}")
-            finally:
-                self.spider_in_progress.pop()
-        log.info(f"close thread: spider")
-
-
-class Storer:
-
-    def store_task(self, stop, last, reset_seed, set_storer):
-
-        inf_name = "StorerInterface"
-        if not issubclass_cobweb_inf(self.__class__, inf_name):
-            return None
-
-        if not getattr(self, "store", None):
-            raise Exception("not have store function!")
-
-        storer_name = self.__class__.__name__ + self.table
-        store_key_id = md5(storer_name.encode()).hexdigest()
-
-        while not stop.is_set():
-
-            if last.is_set() or self.queue.length >= self.length:
-                seeds, data_list = [], []
-
-                while True:
-                    data = self.queue.pop()
-                    if not data:
-                        break
-                    if isinstance(data, Seed):
-                        seeds.append(data)
-                        if len(data_list) >= self.length:
-                            break
-                        continue
-                    data_list.append(data)
-
-                if data_list:
-                    if self.store(data_list):
-                        set_storer(store_key_id, seeds)
-                    else:
-                        reset_seed(seeds)
-                    continue
-
-            time.sleep(3)
-
-        log.info(f"close thread: {storer_name}")