cobweb-launcher 0.1.8__py3-none-any.whl → 1.2.42__py3-none-any.whl

cobweb/base/seed.py

@@ -0,0 +1,114 @@
+import json
+import time
+import hashlib
+
+
+class SeedParams:
+
+    def __init__(self, retry, priority, seed_version, seed_status=None):
+        self.retry = retry or 0
+        self.priority = priority or 300
+        self.seed_version = seed_version or int(time.time())
+        self.seed_status = seed_status
+
+
+class Seed:
+
+    __SEED_PARAMS__ = [
+        "retry",
+        "priority",
+        "seed_version",
+        "seed_status"
+    ]
+
+    def __init__(
+            self,
+            seed,
+            sid=None,
+            retry=None,
+            priority=None,
+            seed_version=None,
+            seed_status=None,
+            **kwargs
+    ):
+        if any(isinstance(seed, t) for t in (str, bytes)):
+            try:
+                item = json.loads(seed)
+                self._init_seed(item)
+            except json.JSONDecodeError:
+                self.__setattr__("url", seed)
+        elif isinstance(seed, dict):
+            self._init_seed(seed)
+        else:
+            raise TypeError(Exception(
+                f"seed type error, "
+                f"must be str or dict! "
+                f"seed: {seed}"
+            ))
+
+        seed_params = {
+            "retry": retry,
+            "priority": priority,
+            "seed_version": seed_version,
+            "seed_status": seed_status,
+        }
+
+        if kwargs:
+            self._init_seed(kwargs)
+            seed_params.update({
+                k:v for k, v in kwargs.items()
+                if k in self.__SEED_PARAMS__
+            })
+        if sid or not getattr(self, "sid", None):
+            self._init_id(sid)
+        self.params = SeedParams(**seed_params)
+
+    def __getattr__(self, name):
+        return None
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __str__(self):
+        return json.dumps(self.__dict__, ensure_ascii=False)
+
+    def __repr__(self):
+        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
+        return f'{self.__class__.__name__}({", ".join(chars)})'
+
+    def _init_seed(self, seed_info:dict):
+        for k, v in seed_info.items():
+            if k not in self.__SEED_PARAMS__:
+                self.__setattr__(k, v)
+
+    def _init_id(self, sid):
+        if not sid:
+            sid = hashlib.md5(self.to_string.encode()).hexdigest()
+        self.__setattr__("sid", sid)
+
+    @property
+    def to_dict(self) -> dict:
+        seed = self.__dict__.copy()
+        if seed.get("params"):
+            del seed["params"]
+        return seed
+
+    @property
+    def to_string(self) -> str:
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+
+    @property
+    def get_all(self):
+        return json.dumps(
+            self.__dict__,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+

cobweb/constant.py

@@ -0,0 +1,94 @@
+
+class CrawlerModel:
+
+    default = "cobweb.crawlers.Crawler"
+    file_air = "cobweb.crawlers.FileCrawlerAir"
+    file_pro = "cobweb.crawlers.FileCrawlerPro"
+
+
+class LauncherModel:
+    task = "launcher model: task"
+    resident = "launcher model: resident"
+
+
+class DownloadModel:
+    common = "download model: common"
+    file = "download model: file"
+
+
+class LogModel:
+    simple = "log model: simple"
+    common = "log model: common"
+    detailed = "log model: detailed"
+
+
+class DealModel:
+    fail = "deal model: fail"
+    done = "deal model: done"
+    poll = "deal model: poll"
+
+
+class LogTemplate:
+
+    console_item = """
+    ----------------------- start - console pipeline -----------------
+            种子详情 \n{seed_detail}
+            解析详情 \n{parse_detail}
+    ----------------------- end  - console pipeline ------------------
+    """
+
+    launcher_air_polling = """
+    ----------------------- start - 轮训日志: {task} -----------------
+            内存队列 
+                种子数:  {doing_len}
+                待消费:  {todo_len}
+                已消费:  {done_len}
+            存储队列 
+                待上传:  {upload_len}
+    ----------------------- end  - 轮训日志: {task} ------------------
+    """
+
+    launcher_pro_polling = """
+----------------------- start - 轮训日志: {task} -----------------
+        内存队列 
+            种子数:  {doing_len}
+            待消费:  {todo_len}
+            已消费:  {done_len}
+        redis队列 
+            种子数:  {redis_seed_count}
+            待消费:  {redis_todo_len}
+            消费中:  {redis_doing_len}
+        存储队列 
+            待上传:  {upload_len}
+----------------------- end  - 轮训日志: {task} ------------------
+"""
+
+    download_exception = """
+----------------------- download exception -----------------------
+        种子详情 \n{detail}
+        种子参数
+            retry         :    {retry}
+            priority      :    {priority}
+            seed_version  :    {seed_version}
+            identifier    :    {identifier}
+        exception
+            msg           :    {exception}
+------------------------------------------------------------------
+"""
+
+    download_info = """
+------------------------ download info ---------------------------
+        种子详情 \n{detail}
+        种子参数
+            retry         :    {retry}
+            priority      :    {priority}
+            seed_version  :    {seed_version}
+            identifier    :    {identifier}
+        response 
+            status        :    {status} \n{response}
+------------------------------------------------------------------
+"""
+
+    @staticmethod
+    def log_info(item: dict) -> str:
+        return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

cobweb/crawlers/__init__.py

@@ -0,0 +1 @@
+from .crawler import Crawler

cobweb/crawlers/base_crawler.py

@@ -0,0 +1,144 @@
+import threading
+import time
+import traceback
+
+from inspect import isgenerator
+from typing import Union, Callable, Mapping
+
+from cobweb.base import Queue, Seed, BaseItem, Request, Response, logger
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.utils import download_log_info
+from cobweb import setting
+
+
+class Crawler(threading.Thread):
+
+    def __init__(
+            self,
+            upload_queue: Queue,
+            custom_func: Union[Mapping[str, Callable]],
+            launcher_queue: Union[Mapping[str, Queue]],
+    ):
+        super().__init__()
+
+        self.upload_queue = upload_queue
+        for func_name, _callable in custom_func.items():
+            if isinstance(_callable, Callable):
+                self.__setattr__(func_name, _callable)
+
+        self.launcher_queue = launcher_queue
+
+        self.spider_thread_num = setting.SPIDER_THREAD_NUM
+        self.max_retries = setting.SPIDER_MAX_RETRIES
+
+    @staticmethod
+    def request(seed: Seed) -> Union[Request, BaseItem]:
+        stream = True if setting.DOWNLOAD_MODEL else False
+        yield Request(seed.url, seed, stream=stream, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> BaseItem:
+        pass
+
+    def get_seed(self) -> Seed:
+        return self.launcher_queue['todo'].pop()
+
+    def distribute(self, item, seed):
+        if isinstance(item, BaseItem):
+            self.upload_queue.push(item)
+        elif isinstance(item, Seed):
+            self.launcher_queue['new'].push(item)
+        elif isinstance(item, str) and item == DealModel.poll:
+            self.launcher_queue['todo'].push(seed)
+        elif isinstance(item, str) and item == DealModel.done:
+            self.launcher_queue['done'].push(seed)
+        elif isinstance(item, str) and item == DealModel.fail:
+            seed.params.seed_status = DealModel.fail
+            self.launcher_queue['done'].push(seed)
+        else:
+            raise TypeError("yield value type error!")
+
+    def spider(self):
+        while True:
+            seed = self.get_seed()
+
+            if not seed:
+                continue
+
+            elif seed.params.retry >= self.max_retries:
+                seed.params.seed_status = DealModel.fail
+                self.launcher_queue['done'].push(seed)
+                continue
+
+            seed_detail_log_info = download_log_info(seed.to_dict)
+
+            try:
+                request_iterators = self.request(seed)
+
+                if not isgenerator(request_iterators):
+                    raise TypeError("request function isn't a generator!")
+
+                iterator_status = False
+
+                for request_item in request_iterators:
+
+                    iterator_status = True
+
+                    if isinstance(request_item, Request):
+                        iterator_status = False
+                        download_iterators = self.download(request_item)
+                        if not isgenerator(download_iterators):
+                            raise TypeError("download function isn't a generator")
+
+                        for download_item in download_iterators:
+                            iterator_status = True
+                            if isinstance(download_item, Response):
+                                iterator_status = False
+                                logger.info(LogTemplate.download_info.format(
+                                    detail=seed_detail_log_info,
+                                    retry=seed.params.retry,
+                                    priority=seed.params.priority,
+                                    seed_version=seed.params.seed_version,
+                                    identifier=seed.identifier or "",
+                                    status=download_item.response,
+                                    response=download_log_info(download_item.to_dict)
+                                ))
+                                parse_iterators = self.parse(download_item)
+                                if not isgenerator(parse_iterators):
+                                    raise TypeError("parse function isn't a generator")
+                                for parse_item in parse_iterators:
+                                    iterator_status = True
+                                    if isinstance(parse_item, Response):
+                                        raise TypeError("upload_item can't be a Response instance")
+                                    self.distribute(parse_item, seed)
+                            else:
+                                self.distribute(download_item, seed)
+                    else:
+                        self.distribute(request_item, seed)
+
+                if not iterator_status:
+                    raise ValueError("request/download/parse function yield value error!")
+
+            except Exception as e:
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info,
+                    retry=seed.params.retry,
+                    priority=seed.params.priority,
+                    seed_version=seed.params.seed_version,
+                    identifier=seed.identifier or "",
+                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                ))
+                seed.params.retry += 1
+                self.launcher_queue['todo'].push(seed)
+            finally:
+                time.sleep(0.1)
+
+    def run(self):
+        for index in range(self.spider_thread_num):
+            threading.Thread(name=f"spider_{index}", target=self.spider).start()
+

cobweb/crawlers/crawler.py

@@ -0,0 +1,212 @@
+import json
+import threading
+import time
+import traceback
+from inspect import isgenerator
+from typing import Union, Callable, Mapping
+from urllib.parse import urlparse
+
+import urllib3
+from requests import HTTPError, Response as Res
+
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.base import (
+    Seed,
+    BaseItem, 
+    Request, 
+    Response, 
+    ConsoleItem,
+    logger
+)
+from cobweb.utils import LoghubDot
+
+
+class Crawler(threading.Thread):
+
+    def __init__(
+            self,
+            task: str,
+            project: str,
+            stop: threading.Event,
+            pause: threading.Event,
+            # launcher_queue: Union[Mapping[str, Queue]],
+            get_seed: Callable,
+            set_seed: Callable,
+            add_seed: Callable,
+            delete_seed: Callable,
+            upload_data: Callable,
+            custom_func: Union[Mapping[str, Callable]],
+            thread_num: int,
+            max_retries: int,
+            time_sleep: int,
+    ):
+        super().__init__()
+        self.task = task
+        self.project = project
+        self._stop = stop
+        self._pause = pause
+        self._get_seed = get_seed
+        self._set_seed = set_seed
+        self._add_seed = add_seed
+        self._delete_seed = delete_seed
+        self._upload_data = upload_data
+
+        for func_name, _callable in custom_func.items():
+            if isinstance(_callable, Callable):
+                self.__setattr__(func_name, _callable)
+
+        self.thread_num = thread_num
+        self.time_sleep = time_sleep
+        self.max_retries = max_retries
+
+        self.loghub_dot = LoghubDot()
+
+    @staticmethod
+    def request(seed: Seed) -> Union[Request, BaseItem]:
+        yield Request(seed.url, seed, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> BaseItem:
+        upload_item = item.to_dict
+        upload_item["text"] = item.response.text
+        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
+
+    # def get_seed(self) -> Seed:
+    #     return self._todo.pop()
+
+    def distribute(self, item, seed):
+        if isinstance(item, BaseItem):
+            self._upload_data(item)
+        elif isinstance(item, Seed):
+            self._add_seed(item)
+        elif isinstance(item, str) and item == DealModel.poll:
+            self._set_seed(seed)
+        elif isinstance(item, str) and item == DealModel.done:
+            self._delete_seed(seed)
+        elif isinstance(item, str) and item == DealModel.fail:
+            seed.params.seed_status = DealModel.fail
+            self._delete_seed(seed)
+        else:
+            raise TypeError("yield value type error!")
+
+    def spider(self):
+        while not self._stop.is_set():
+
+            seed = self._get_seed()
+
+            if not seed:
+                time.sleep(1)
+                continue
+
+            elif seed.params.retry > self.max_retries:
+                seed.params.seed_status = DealModel.fail
+                self._delete_seed(seed)
+                continue
+
+            seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
+
+            try:
+                request_iterators = self.request(seed)
+
+                if not isgenerator(request_iterators):
+                    raise TypeError("request function isn't a generator!")
+
+                iterator_status = False
+
+                for request_item in request_iterators:
+
+                    iterator_status = True
+
+                    if isinstance(request_item, Request):
+                        iterator_status = False
+                        start_time = time.time()
+                        download_iterators = self.download(request_item)
+                        if not isgenerator(download_iterators):
+                            raise TypeError("download function isn't a generator")
+
+                        for download_item in download_iterators:
+                            iterator_status = True
+                            if isinstance(download_item, Response):
+                                iterator_status = False
+                                logger.info(LogTemplate.download_info.format(
+                                    detail=seed_detail_log_info,
+                                    retry=seed.params.retry,
+                                    priority=seed.params.priority,
+                                    seed_version=seed.params.seed_version,
+                                    identifier=seed.identifier or "",
+                                    status=download_item.response,
+                                    response=LogTemplate.log_info(download_item.to_dict)
+                                ))
+                                if isinstance(download_item.response, Res):
+                                    end_time = time.time()
+                                    self.loghub_dot.build(
+                                        topic=urlparse(download_item.response.request.url).netloc,
+                                        data_size=int(download_item.response.headers.get("content-length", 0)),
+                                        cost_time=end_time - start_time, status = 200,
+                                        url=download_item.response.url,
+                                    )
+                                parse_iterators = self.parse(download_item)
+                                if not isgenerator(parse_iterators):
+                                    raise TypeError("parse function isn't a generator")
+                                for parse_item in parse_iterators:
+                                    iterator_status = True
+                                    if isinstance(parse_item, Response):
+                                        raise TypeError("upload_item can't be a Response instance")
+                                    self.distribute(parse_item, seed)
+                            else:
+                                self.distribute(download_item, seed)
+                    else:
+                        self.distribute(request_item, seed)
+
+                if not iterator_status:
+                    raise ValueError("request/download/parse function yield value error!")
+            except (HTTPError, urllib3.exceptions.HTTPError, urllib3.exceptions.PoolError) as e:
+                exception_msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                url = seed.url
+                status = str(e)
+                if getattr(e, "response", None) and isinstance(e.response, Res):
+                    url = e.response.request.url
+                    status = e.response.status_code
+                self.loghub_dot.build(
+                    topic=urlparse(url).netloc,
+                    data_size=-1, cost_time=-1,
+                    status=status, url=url,
+                    msg=exception_msg
+                )
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info,
+                    retry=seed.params.retry,
+                    priority=seed.params.priority,
+                    seed_version=seed.params.seed_version,
+                    identifier=seed.identifier or "",
+                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                ))
+                seed.params.retry += 1
+                self._set_seed(seed)
+                # time.sleep(self.time_sleep * seed.params.retry)
+            except Exception as e:
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info,
+                    retry=seed.params.retry,
+                    priority=seed.params.priority,
+                    seed_version=seed.params.seed_version,
+                    identifier=seed.identifier or "",
+                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                ))
+                seed.params.retry += 1
+                # self._todo.push(seed)
+                self._set_seed(seed)
+                # time.sleep(self.time_sleep * seed.params.retry)
+            finally:
+                time.sleep(0.1)
+        logger.info("spider thread close")
+
+    def run(self):
+        for index in range(self.thread_num):
+            threading.Thread(name=f"spider_{index}", target=self.spider).start()
+

cobweb/crawlers/file_crawler.py

@@ -0,0 +1,98 @@
+import os
+from typing import Union
+from cobweb import setting
+from cobweb.utils import OssUtil
+from cobweb.crawlers import Crawler
+from cobweb.base import Seed, BaseItem, Request, Response
+from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
+
+
+oss_util = OssUtil(is_path_style=bool(int(os.getenv("PRIVATE_LINK", 0))))
+
+
+class FileCrawlerAir(Crawler):
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        seed_dict = item.seed.to_dict
+        seed_dict["bucket_name"] = oss_util.bucket
+        try:
+            seed_dict["oss_path"] = key = item.seed.oss_path or getattr(item, "oss_path")
+
+            if oss_util.exists(key):
+                seed_dict["data_size"] = oss_util.head(key).content_length
+                yield Response(item.seed, "exists", **seed_dict)
+
+            else:
+                seed_dict.setdefault("end", "")
+                seed_dict.setdefault("start", 0)
+
+                if seed_dict["end"] or seed_dict["start"]:
+                    start, end = seed_dict["start"], seed_dict["end"]
+                    item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
+
+                if not item.seed.identifier:
+                    content = b""
+                    chunk_size = oss_util.chunk_size
+                    min_upload_size = oss_util.min_upload_size
+                    seed_dict.setdefault("position", 1)
+
+                    response = item.download()
+
+                    content_type = response.headers.get("content-type", "").split(";")[0]
+                    seed_dict["data_size"] = content_length = int(response.headers.get("content-length", 0))
+
+                    if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
+                        """过滤响应文件类型"""
+                        response.close()
+                        seed_dict["filter"] = True
+                        seed_dict["msg"] = f"response content type is {content_type}"
+                        yield Response(item.seed, response, **seed_dict)
+
+                    elif seed_dict['position'] == 1 and min_upload_size >= content_length > 0:
+                        """过小文件标识返回"""
+                        response.close()
+                        seed_dict["filter"] = True
+                        seed_dict["msg"] = "file size is too small"
+                        yield Response(item.seed, response, **seed_dict)
+
+                    elif seed_dict['position'] == 1 and chunk_size > content_length > min_upload_size:
+                        """小文件直接下载"""
+                        for part_data in response.iter_content(chunk_size):
+                            content += part_data
+                        response.close()
+                        oss_util.put(key, content)
+                        yield Response(item.seed, response, **seed_dict)
+
+                    else:
+                        """中大文件同步分片下载"""
+                        seed_dict.setdefault("upload_id", oss_util.init_part(key).upload_id)
+
+                        for part_data in response.iter_content(chunk_size):
+                            content += part_data
+                            if len(content) >= chunk_size:
+                                upload_data = content[:chunk_size]
+                                content = content[chunk_size:]
+                                oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
+                                seed_dict['start'] += len(upload_data)
+                                seed_dict['position'] += 1
+
+                        response.close()
+
+                        if content:
+                            oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
+                        oss_util.merge(key, seed_dict["upload_id"])
+                        seed_dict["data_size"] = oss_util.head(key).content_length
+                        yield Response(item.seed, response, **seed_dict)
+
+                elif item.seed.identifier == "merge":
+                    oss_util.merge(key, seed_dict["upload_id"])
+                    seed_dict["data_size"] = oss_util.head(key).content_length
+                    yield Response(item.seed, "merge", **seed_dict)
+
+        except OssDBPutPartError:
+            yield Seed(seed_dict)
+        except OssDBMergeError:
+            yield Seed(seed_dict, identifier="merge")
+
+

cobweb/db/__init__.py

@@ -1,2 +1,2 @@
-from . import oss_db, redis_db
-from . import scheduler, storer
+from .redis_db import RedisDB
+from .api_db import ApiDB