cobweb-launcher 1.3.6__py3-none-any.whl → 1.3.7__py3-none-any.whl

cobweb_new/base/request.py

@@ -1,82 +0,0 @@
-import random
-import requests
-
-
-class Request:
-
-    __REQUEST_ATTRS__ = {
-        "params",
-        "headers",
-        "cookies",
-        "data",
-        "json",
-        "files",
-        "auth",
-        "timeout",
-        "proxies",
-        "hooks",
-        "stream",
-        "verify",
-        "cert",
-        "allow_redirects",
-    }
-
-    def __init__(
-            self,
-            url,
-            seed,
-            random_ua=True,
-            check_status_code=True,
-            **kwargs
-    ):
-        self.url = url
-        self.seed = seed
-        self.check_status_code = check_status_code
-        self.request_setting = {}
-
-        for k, v in kwargs.items():
-            if k in self.__class__.__REQUEST_ATTRS__:
-                self.request_setting[k] = v
-                continue
-            self.__setattr__(k, v)
-
-        if not getattr(self, "method", None):
-            self.method = "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
-
-        if random_ua:
-            self._build_header()
-
-    @property
-    def _random_ua(self) -> str:
-        v1 = random.randint(4, 15)
-        v2 = random.randint(3, 11)
-        v3 = random.randint(1, 16)
-        v4 = random.randint(533, 605)
-        v5 = random.randint(1000, 6000)
-        v6 = random.randint(10, 80)
-        user_agent = (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) AppleWebKit/{v4}.{v3} "
-                      f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
-        return user_agent
-
-    def _build_header(self) -> dict:
-        if not self.request_setting.get("headers"):
-            self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
-        elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:
-            self.request_setting["headers"]["user-agent"] = self._random_ua
-
-    def download(self) -> requests.Response:
-        response = requests.request(self.method, self.url, **self.request_setting)
-        if self.check_status_code:
-            response.raise_for_status()
-        return response
-
-    @property
-    def to_dict(self):
-        _dict = self.__dict__.copy()
-        _dict.pop('url')
-        _dict.pop('seed')
-        _dict.pop('check_status_code')
-        _dict.pop('request_setting')
-        return _dict
-
-

cobweb_new/base/response.py

@@ -1,23 +0,0 @@
-
-
-class Response:
-
-    def __init__(
-            self,
-            seed,
-            response,
-            **kwargs
-    ):
-        self.seed = seed
-        self.response = response
-
-        for k, v in kwargs.items():
-            self.__setattr__(k, v)
-
-    @property
-    def to_dict(self):
-        _dict = self.__dict__.copy()
-        _dict.pop('seed')
-        _dict.pop('response')
-        return _dict
-

cobweb_new/base/seed.py

@@ -1,118 +0,0 @@
-import json
-import time
-import hashlib
-
-
-class SeedParams:
-
-    def __init__(self, retry, priority, seed_version, seed_status=None):
-        self.retry = retry or 0
-        self.priority = priority or 300
-        self.seed_version = seed_version or int(time.time())
-        self.seed_status = seed_status
-
-
-class Seed:
-
-    __SEED_PARAMS__ = [
-        "retry",
-        "priority",
-        "seed_version",
-        "seed_status"
-    ]
-
-    def __init__(
-            self,
-            seed,
-            sid=None,
-            retry=None,
-            priority=None,
-            seed_version=None,
-            seed_status=None,
-            **kwargs
-    ):
-        if any(isinstance(seed, t) for t in (str, bytes)):
-            try:
-                item = json.loads(seed)
-                self._init_seed(item)
-            except json.JSONDecodeError:
-                self.__setattr__("url", seed)
-        elif isinstance(seed, dict):
-            self._init_seed(seed)
-        else:
-            raise TypeError(Exception(
-                f"seed type error, "
-                f"must be str or dict! "
-                f"seed: {seed}"
-            ))
-
-        seed_params = {
-            "retry": retry,
-            "priority": priority,
-            "seed_version": seed_version,
-            "seed_status": seed_status,
-        }
-
-        if kwargs:
-            self._init_seed(kwargs)
-            seed_params.update({
-                k:v for k, v in kwargs.items()
-                if k in self.__SEED_PARAMS__
-            })
-        if sid or not getattr(self, "sid", None):
-            self._init_id(sid)
-        self.params = SeedParams(**seed_params)
-
-    def __getattr__(self, name):
-        return None
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-    def __getitem__(self, item):
-        return getattr(self, item)
-
-    def __str__(self):
-        return json.dumps(self.__dict__, ensure_ascii=False)
-
-    def __repr__(self):
-        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
-        return f'{self.__class__.__name__}({", ".join(chars)})'
-
-    def _init_seed(self, seed_info:dict):
-        for k, v in seed_info.items():
-            if k not in self.__SEED_PARAMS__:
-                self.__setattr__(k, v)
-
-    def _init_id(self, sid):
-        if not sid:
-            sid = hashlib.md5(self.to_string.encode()).hexdigest()
-        self.__setattr__("sid", sid)
-
-    @property
-    def to_dict(self) -> dict:
-        seed = self.__dict__.copy()
-        if seed.get("params"):
-            del seed["params"]
-        return seed
-
-    @property
-    def to_string(self) -> str:
-        return json.dumps(
-            self.to_dict,
-            ensure_ascii=False,
-            separators=(",", ":")
-        )
-
-    # @property
-    # def get_all(self):
-    #     return json.dumps(
-    #         self.__dict__,
-    #         ensure_ascii=False,
-    #         separators=(",", ":")
-    #     )
-
-    @property
-    def seed(self):
-        return self
-

cobweb_new/constant.py

@@ -1,105 +0,0 @@
-
-class CrawlerModel:
-
-    default = "cobweb.crawlers.Crawler"
-    file_air = "cobweb.crawlers.FileCrawlerAir"
-    file_pro = "cobweb.crawlers.FileCrawlerPro"
-
-
-class LauncherModel:
-    task = "launcher model: task"
-    resident = "launcher model: resident"
-
-
-class DownloadModel:
-    common = "download model: common"
-    file = "download model: file"
-
-
-class LogModel:
-    simple = "log model: simple"
-    common = "log model: common"
-    detailed = "log model: detailed"
-
-
-class DealModel:
-    fail = "deal model: fail"
-    done = "deal model: done"
-    poll = "deal model: poll"
-
-
-class LogTemplate:
-
-    console_item = """
-    ----------------------- start - console pipeline -----------------
-            种子详情 \n{seed_detail}
-            解析详情 \n{parse_detail}
-    ----------------------- end  - console pipeline ------------------
-    """
-
-    launcher_polling = """
-    ----------------------- start - 轮训日志: {task} -----------------
-            内存队列 
-                种子数:  {doing_len}
-                待消费:  {todo_len}
-                已消费:  {done_len}
-            存储队列 
-                待上传:  {upload_len}
-    ----------------------- end  - 轮训日志: {task} ------------------
-    """
-
-    launcher_air_polling = """
-    ----------------------- start - 轮训日志: {task} -----------------
-            内存队列 
-                种子数:  {doing_len}
-                待消费:  {todo_len}
-                已消费:  {done_len}
-            存储队列 
-                待上传:  {upload_len}
-    ----------------------- end  - 轮训日志: {task} ------------------
-    """
-
-    launcher_pro_polling = """
------------------------ start - 轮训日志: {task} -----------------
-        内存队列 
-            种子数:  {doing_len}
-            待消费:  {todo_len}
-            已消费:  {done_len}
-        redis队列 
-            种子数:  {redis_seed_count}
-            待消费:  {redis_todo_len}
-            消费中:  {redis_doing_len}
-        存储队列 
-            待上传:  {upload_len}
------------------------ end  - 轮训日志: {task} ------------------
-"""
-
-    download_exception = """
------------------------ download exception -----------------------
-        种子详情 \n{detail}
-        种子参数
-            retry         :    {retry}
-            priority      :    {priority}
-            seed_version  :    {seed_version}
-            identifier    :    {identifier}
-        exception
-            msg           :    {exception}
-------------------------------------------------------------------
-"""
-
-    download_info = """
------------------------- download info ---------------------------
-        种子详情 \n{detail}
-        种子参数
-            retry         :    {retry}
-            priority      :    {priority}
-            seed_version  :    {seed_version}
-            identifier    :    {identifier}
-        response 
-            status        :    {status} \n{response}
-------------------------------------------------------------------
-"""
-
-    @staticmethod
-    def log_info(item: dict) -> str:
-        return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

cobweb_new/crawlers/__init__.py

@@ -1 +0,0 @@
-from .crawler import Crawler

cobweb_new/crawlers/crawler-new.py

@@ -1,85 +0,0 @@
-import json
-import time
-import threading
-from typing import Union, Callable, Mapping
-
-import setting
-from cobweb.base import (
-    Seed,
-    BaseItem,
-    Request,
-    Response,
-    ConsoleItem,
-    decorators,
-    TaskQueue,
-    logger
-)
-from constant import DealModel
-
-
-class Crawler(threading.Thread):
-
-    def __init__(self, custom_func: Union[Mapping[str, Callable]]):
-        super().__init__()
-
-        for func_name, _callable in custom_func.items():
-            if isinstance(_callable, Callable):
-                self.__setattr__(func_name, _callable)
-
-        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
-        self.request_queue_size = setting.REQUEST_QUEUE_SIZE
-        self.download_queue_size = setting.DOWNLOAD_QUEUE_SIZE
-        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
-
-    @staticmethod
-    def request(seed: Seed) -> Union[Request, BaseItem]:
-        yield Request(seed.url, seed, timeout=5)
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        response = item.download()
-        yield Response(item.seed, response, **item.to_dict)
-
-    @staticmethod
-    def parse(item: Response) -> BaseItem:
-        upload_item = item.to_dict
-        upload_item["text"] = item.response.text
-        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
-
-    # @decorators.add_thread()
-    @decorators.pause
-    def build_request_item(self):
-        thread_sleep = 0.1
-        if TaskQueue.REQUEST.length >= self.request_queue_size:
-            thread_sleep = 5
-        elif seed := TaskQueue.TODO.pop():
-            if seed.params.retry > self.spider_max_retries:
-                seed.params.seed_status = DealModel.fail
-            else:
-                TaskQueue.process_task(seed, self.request)
-            TaskQueue.DELETE.push(seed)
-        time.sleep(thread_sleep)
-
-    # @decorators.add_thread(num=setting.SPIDER_THREAD_NUM)
-    @decorators.pause
-    def build_download_item(self):
-        thread_sleep = 0.1
-        if TaskQueue.DOWNLOAD.length >= self.download_queue_size:
-            logger.info(f"download queue is full, sleep {thread_sleep}s")
-            thread_sleep = 5
-        elif request_item := TaskQueue.REQUEST.pop():
-            TaskQueue.process_task(request_item, self.download)
-        time.sleep(thread_sleep)
-
-    # @decorators.add_thread()
-    @decorators.pause
-    def build_parse_item(self):
-        thread_sleep = 0.1
-        if TaskQueue.UPLOAD.length >= self.upload_queue_size:
-            logger.info(f"upload queue is full, sleep {thread_sleep}s")
-            thread_sleep = 5
-        if response_item := TaskQueue.RESPONSE.pop():
-            TaskQueue.process_task(response_item, self.parse)
-        time.sleep(thread_sleep)
-
-

cobweb_new/crawlers/crawler.py

@@ -1,170 +0,0 @@
-import json
-import threading
-import time
-import traceback
-from inspect import isgenerator
-from typing import Union, Callable, Mapping
-
-from cobweb.constant import DealModel, LogTemplate
-from cobweb.base import (
-    Queue, 
-    Seed, 
-    BaseItem, 
-    Request, 
-    Response, 
-    ConsoleItem,
-    logger
-)
-
-
-class Crawler(threading.Thread):
-
-    def __init__(
-            self,
-            stop: threading.Event,
-            pause: threading.Event,
-            # launcher_queue: Union[Mapping[str, Queue]],
-            get_seed: Callable,
-            set_seed: Callable,
-            add_seed: Callable,
-            delete_seed: Callable,
-            upload_data: Callable,
-            custom_func: Union[Mapping[str, Callable]],
-            thread_num: int,
-            max_retries: int,
-            time_sleep: int,
-    ):
-        super().__init__()
-
-        self._stop = stop
-        self._pause = pause
-        self._get_seed = get_seed
-        self._set_seed = set_seed
-        self._add_seed = add_seed
-        self._delete_seed = delete_seed
-        self._upload_data = upload_data
-
-        for func_name, _callable in custom_func.items():
-            if isinstance(_callable, Callable):
-                self.__setattr__(func_name, _callable)
-
-        self.thread_num = thread_num
-        self.time_sleep = time_sleep
-        self.max_retries = max_retries
-
-    @staticmethod
-    def request(seed: Seed) -> Union[Request, BaseItem]:
-        yield Request(seed.url, seed, timeout=5)
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        response = item.download()
-        yield Response(item.seed, response, **item.to_dict)
-
-    @staticmethod
-    def parse(item: Response) -> BaseItem:
-        upload_item = item.to_dict
-        upload_item["text"] = item.response.text
-        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
-
-    # def get_seed(self) -> Seed:
-    #     return self._todo.pop()
-
-    def distribute(self, item, seed):
-        if isinstance(item, BaseItem):
-            self._upload_data(item)
-        elif isinstance(item, Seed):
-            self._add_seed(item)
-        elif isinstance(item, str) and item == DealModel.poll:
-            self._set_seed(seed)
-        elif isinstance(item, str) and item == DealModel.done:
-            self._delete_seed(seed)
-        elif isinstance(item, str) and item == DealModel.fail:
-            seed.params.seed_status = DealModel.fail
-            self._delete_seed(seed)
-        else:
-            raise TypeError("yield value type error!")
-
-    def spider(self):
-        while not self._stop.is_set():
-
-            seed = self._get_seed()
-
-            if not seed:
-                time.sleep(1)
-                continue
-
-            elif seed.params.retry > self.max_retries:
-                seed.params.seed_status = DealModel.fail
-                self._delete_seed(seed)
-                continue
-
-            seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
-
-            try:
-                request_iterators = self.request(seed)
-
-                if not isgenerator(request_iterators):
-                    raise TypeError("request function isn't a generator!")
-
-                iterator_status = False
-
-                for request_item in request_iterators:
-
-                    iterator_status = True
-
-                    if isinstance(request_item, Request):
-                        iterator_status = False
-                        download_iterators = self.download(request_item)
-                        if not isgenerator(download_iterators):
-                            raise TypeError("download function isn't a generator")
-
-                        for download_item in download_iterators:
-                            iterator_status = True
-                            if isinstance(download_item, Response):
-                                iterator_status = False
-                                logger.info(LogTemplate.download_info.format(
-                                    detail=seed_detail_log_info,
-                                    retry=seed.params.retry,
-                                    priority=seed.params.priority,
-                                    seed_version=seed.params.seed_version,
-                                    identifier=seed.identifier or "",
-                                    status=download_item.response,
-                                    response=LogTemplate.log_info(download_item.to_dict)
-                                ))
-                                parse_iterators = self.parse(download_item)
-                                if not isgenerator(parse_iterators):
-                                    raise TypeError("parse function isn't a generator")
-                                for parse_item in parse_iterators:
-                                    iterator_status = True
-                                    if isinstance(parse_item, Response):
-                                        raise TypeError("upload_item can't be a Response instance")
-                                    self.distribute(parse_item, seed)
-                            else:
-                                self.distribute(download_item, seed)
-                    else:
-                        self.distribute(request_item, seed)
-
-                if not iterator_status:
-                    raise ValueError("request/download/parse function yield value error!")
-            except Exception as e:
-                logger.info(LogTemplate.download_exception.format(
-                    detail=seed_detail_log_info,
-                    retry=seed.params.retry,
-                    priority=seed.params.priority,
-                    seed_version=seed.params.seed_version,
-                    identifier=seed.identifier or "",
-                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                ))
-                seed.params.retry += 1
-                # self._todo.push(seed)
-                self._set_seed(seed)
-                time.sleep(self.time_sleep * seed.params.retry)
-            finally:
-                time.sleep(0.1)
-        logger.info("spider thread close")
-
-    def run(self):
-        for index in range(self.thread_num):
-            threading.Thread(name=f"spider_{index}", target=self.spider).start()
-

cobweb_new/db/__init__.py

@@ -1,2 +0,0 @@
-from .redis_db import RedisDB
-from .api_db import ApiDB

cobweb_new/db/api_db.py

@@ -1,82 +0,0 @@
-import os
-import json
-import requests
-
-
-class ApiDB:
-
-    def __init__(self, host=None, **kwargs):
-        self.host = host or os.getenv("REDIS_API_HOST", "http://127.0.0.1:4396")
-
-    def _get_response(self, api, params: dict = None):
-        try:
-            url = self.host + api
-            response = requests.get(url, params=params)
-            json_data = response.json()
-            response.close()
-            return json_data["data"]
-        except:
-            return None
-
-    def _post_response(self, api, params: dict = None, data: dict = None):
-        try:
-            url = self.host + api
-            headers = {"Content-Type": "application/json"}
-            response = requests.post(url, headers=headers, params=params, data=json.dumps(data))
-            json_data = response.json()
-            response.close()
-            return json_data["data"]
-        except:
-            return None
-
-    def get(self, name):
-        return self._get_response(api="/get", params=dict(name=name))
-
-    def setnx(self, name, value=""):
-        return self._get_response(api="/setnx", params=dict(name=name, value=value))
-
-    def setex(self, name, t, value=""):
-        return self._get_response(api="/setex", params=dict(name=name, value=value, t=t))
-
-    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        return self._get_response(api="/expire", params=dict(name=name, t=t, nx=nx, xx=xx, gt=gt, lt=lt))
-
-    def ttl(self, name):
-        return self._get_response(api="/ttl", params=dict(name=name))
-
-    def delete(self, name):
-        return self._get_response(api="/delete", params=dict(name=name))
-
-    def exists(self, name):
-        return self._get_response(api="/exists", params=dict(name=name))
-
-    def incrby(self, name, value):
-        return self._get_response(api="/incrby", params=dict(name=name, value=value))
-
-    def zcard(self, name) -> bool:
-        return self._get_response(api="/zcard", params=dict(name=name))
-
-    def zadd(self, name, item: dict, **kwargs):
-        return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
-
-    def zrem(self, name, *values):
-        return self._post_response(api="/zrem", data=dict(name=name, values=values))
-
-    def zcount(self, name, _min, _max):
-        return self._get_response(api="/zcount", params=dict(name=name, min=_min, max=_max))
-
-    def lock(self, name, t=15) -> bool:
-        return self._get_response(api="/lock", params=dict(name=name, t=t))
-
-    def auto_incr(self, name, t=15, limit=1000) -> bool:
-        return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
-
-    def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
-        return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
-
-    def done(self, name: list, *values):
-        return self._post_response(api="/done", data=dict(name=name, values=values))
-
-
-
-