cobweb-launcher 1.2.25__tar.gz → 1.2.27__tar.gz

{cobweb-launcher-1.2.25/cobweb_launcher.egg-info → cobweb-launcher-1.2.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.25
+Version: 1.2.27
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb-launcher-1.2.27/cobweb/crawlers/crawler.py

@@ -0,0 +1,183 @@
+import json
+import threading
+import time
+import traceback
+from inspect import isgenerator
+from typing import Union, Callable, Mapping
+
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.base import (
+    Queue, 
+    Seed, 
+    BaseItem, 
+    Request, 
+    Response, 
+    ConsoleItem,
+    logger
+)
+from cobweb.utils import LoghubDot
+
+
+class Crawler(threading.Thread):
+
+    def __init__(
+            self,
+            task: str,
+            project: str,
+            stop: threading.Event,
+            pause: threading.Event,
+            # launcher_queue: Union[Mapping[str, Queue]],
+            get_seed: Callable,
+            set_seed: Callable,
+            add_seed: Callable,
+            delete_seed: Callable,
+            upload_data: Callable,
+            custom_func: Union[Mapping[str, Callable]],
+            thread_num: int,
+            max_retries: int,
+            time_sleep: int,
+    ):
+        super().__init__()
+        self.task = task
+        self.project = project
+        self._stop = stop
+        self._pause = pause
+        self._get_seed = get_seed
+        self._set_seed = set_seed
+        self._add_seed = add_seed
+        self._delete_seed = delete_seed
+        self._upload_data = upload_data
+
+        for func_name, _callable in custom_func.items():
+            if isinstance(_callable, Callable):
+                self.__setattr__(func_name, _callable)
+
+        self.thread_num = thread_num
+        self.time_sleep = time_sleep
+        self.max_retries = max_retries
+
+        self.loghub_dot = LoghubDot()
+
+    @staticmethod
+    def request(seed: Seed) -> Union[Request, BaseItem]:
+        yield Request(seed.url, seed, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> BaseItem:
+        upload_item = item.to_dict
+        upload_item["text"] = item.response.text
+        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
+
+    # def get_seed(self) -> Seed:
+    #     return self._todo.pop()
+
+    def distribute(self, item, seed):
+        if isinstance(item, BaseItem):
+            self._upload_data(item)
+        elif isinstance(item, Seed):
+            self._add_seed(item)
+        elif isinstance(item, str) and item == DealModel.poll:
+            self._set_seed(seed)
+        elif isinstance(item, str) and item == DealModel.done:
+            self._delete_seed(seed)
+        elif isinstance(item, str) and item == DealModel.fail:
+            seed.params.seed_status = DealModel.fail
+            self._delete_seed(seed)
+        else:
+            raise TypeError("yield value type error!")
+
+    def spider(self):
+        while not self._stop.is_set():
+
+            seed = self._get_seed()
+
+            if not seed:
+                time.sleep(1)
+                continue
+
+            elif seed.params.retry > self.max_retries:
+                seed.params.seed_status = DealModel.fail
+                self._delete_seed(seed)
+                continue
+
+            seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
+
+            try:
+                request_iterators = self.request(seed)
+
+                if not isgenerator(request_iterators):
+                    raise TypeError("request function isn't a generator!")
+
+                iterator_status = False
+
+                for request_item in request_iterators:
+
+                    iterator_status = True
+
+                    if isinstance(request_item, Request):
+                        iterator_status = False
+                        start_time = time.time()
+                        download_iterators = self.download(request_item)
+                        if not isgenerator(download_iterators):
+                            raise TypeError("download function isn't a generator")
+
+                        for download_item in download_iterators:
+                            iterator_status = True
+                            if isinstance(download_item, Response):
+                                iterator_status = False
+                                logger.info(LogTemplate.download_info.format(
+                                    detail=seed_detail_log_info,
+                                    retry=seed.params.retry,
+                                    priority=seed.params.priority,
+                                    seed_version=seed.params.seed_version,
+                                    identifier=seed.identifier or "",
+                                    status=download_item.response,
+                                    response=LogTemplate.log_info(download_item.to_dict)
+                                ))
+                                if isinstance(download_item, Response):
+                                    end_time = time.time()
+                                    self.loghub_dot.build(topic=f"{self.project}:{self.task}", data={
+                                        "cost_time": end_time - start_time,
+                                        **download_item.to_dict
+                                    })
+                                parse_iterators = self.parse(download_item)
+                                if not isgenerator(parse_iterators):
+                                    raise TypeError("parse function isn't a generator")
+                                for parse_item in parse_iterators:
+                                    iterator_status = True
+                                    if isinstance(parse_item, Response):
+                                        raise TypeError("upload_item can't be a Response instance")
+                                    self.distribute(parse_item, seed)
+                            else:
+                                self.distribute(download_item, seed)
+                    else:
+                        self.distribute(request_item, seed)
+
+                if not iterator_status:
+                    raise ValueError("request/download/parse function yield value error!")
+            except Exception as e:
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info,
+                    retry=seed.params.retry,
+                    priority=seed.params.priority,
+                    seed_version=seed.params.seed_version,
+                    identifier=seed.identifier or "",
+                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                ))
+                seed.params.retry += 1
+                # self._todo.push(seed)
+                self._set_seed(seed)
+                time.sleep(self.time_sleep * seed.params.retry)
+            finally:
+                time.sleep(0.1)
+        logger.info("spider thread close")
+
+    def run(self):
+        for index in range(self.thread_num):
+            threading.Thread(name=f"spider_{index}", target=self.spider).start()
+

{cobweb-launcher-1.2.25 → cobweb-launcher-1.2.27}/cobweb/launchers/launcher.py

@@ -4,6 +4,8 @@ import threading
 import importlib
 from functools import wraps
 
+from mypyc.doc.conf import project
+
 from cobweb import setting
 from cobweb.base import Seed, Queue, logger
 from cobweb.utils.tools import dynamic_load_class
@@ -186,6 +188,7 @@ class Launcher(threading.Thread):
         self.start_seeds()
 
         self._Crawler(
+            task=self.task, project=self.project,
             stop=self._stop, pause=self._pause,
             # launcher_queue=self.__LAUNCHER_QUEUE__,
             get_seed=self._get_seed,

{cobweb-launcher-1.2.25 → cobweb-launcher-1.2.27}/cobweb/utils/__init__.py

@@ -1,4 +1,5 @@
 from .oss import OssUtil
 from .tools import *
 from .bloom import BloomFilter
+from .dotting import LoghubDot
 

cobweb-launcher-1.2.27/cobweb/utils/dotting.py

@@ -0,0 +1,28 @@
+import json
+
+from aliyun.log import LogClient, PutLogsRequest, LogItem
+
+import setting
+
+
+class LoghubDot:
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = LogClient(**setting.LOGHUB_CONFIG)
+
+    def build(self, topic, data):
+        log_item = LogItem()
+        for key, value in data.items():
+            if not isinstance(value, str):
+                data[key] = json.dumps(value, ensure_ascii=False)
+        contents = sorted(data.items())
+        log_item.set_contents(contents)
+        request = PutLogsRequest(
+            project=setting.LOGHUB_PROJECT,
+            logstore="cobweb_log",
+            topic=topic,
+            logitems=contents,
+            compress=True
+        )
+        self.client.put_logs(request=request)

{cobweb-launcher-1.2.25 → cobweb-launcher-1.2.27/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.25
+Version: 1.2.27
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb-launcher-1.2.27/cobweb_launcher.egg-info/SOURCES.txt

@@ -0,0 +1,74 @@
+LICENSE
+README.md
+setup.py
+cobweb/__init__.py
+cobweb/constant.py
+cobweb/setting.py
+cobweb/base/__init__.py
+cobweb/base/common_queue.py
+cobweb/base/decorators.py
+cobweb/base/item.py
+cobweb/base/log.py
+cobweb/base/request.py
+cobweb/base/response.py
+cobweb/base/seed.py
+cobweb/crawlers/__init__.py
+cobweb/crawlers/crawler.py
+cobweb/db/__init__.py
+cobweb/db/api_db.py
+cobweb/db/redis_db.py
+cobweb/exceptions/__init__.py
+cobweb/exceptions/oss_db_exception.py
+cobweb/launchers/__init__.py
+cobweb/launchers/launcher.py
+cobweb/launchers/launcher_air.py
+cobweb/launchers/launcher_api.py
+cobweb/launchers/launcher_pro.py
+cobweb/pipelines/__init__.py
+cobweb/pipelines/pipeline.py
+cobweb/pipelines/pipeline_console.py
+cobweb/pipelines/pipeline_loghub.py
+cobweb/utils/__init__.py
+cobweb/utils/bloom.py
+cobweb/utils/dotting.py
+cobweb/utils/oss.py
+cobweb/utils/tools.py
+cobweb_launcher.egg-info/PKG-INFO
+cobweb_launcher.egg-info/SOURCES.txt
+cobweb_launcher.egg-info/dependency_links.txt
+cobweb_launcher.egg-info/requires.txt
+cobweb_launcher.egg-info/top_level.txt
+cobweb_new/__init__.py
+cobweb_new/constant.py
+cobweb_new/setting.py
+cobweb_new/base/__init__.py
+cobweb_new/base/common_queue.py
+cobweb_new/base/decorators.py
+cobweb_new/base/item.py
+cobweb_new/base/log.py
+cobweb_new/base/request.py
+cobweb_new/base/response.py
+cobweb_new/base/seed.py
+cobweb_new/crawlers/__init__.py
+cobweb_new/crawlers/crawler-new.py
+cobweb_new/crawlers/crawler.py
+cobweb_new/db/__init__.py
+cobweb_new/db/api_db.py
+cobweb_new/db/redis_db.py
+cobweb_new/exceptions/__init__.py
+cobweb_new/exceptions/oss_db_exception.py
+cobweb_new/launchers/__init__.py
+cobweb_new/launchers/launcher.py
+cobweb_new/launchers/launcher_air.py
+cobweb_new/launchers/launcher_api.py
+cobweb_new/launchers/launcher_pro.py
+cobweb_new/launchers/tesss.py
+cobweb_new/pipelines/__init__.py
+cobweb_new/pipelines/pipeline.py
+cobweb_new/pipelines/pipeline_console.py
+cobweb_new/pipelines/pipeline_loghub.py
+cobweb_new/utils/__init__.py
+cobweb_new/utils/bloom.py
+cobweb_new/utils/oss.py
+cobweb_new/utils/tools.py
+test/test.py

cobweb-launcher-1.2.27/cobweb_launcher.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+cobweb
+cobweb_new

cobweb-launcher-1.2.27/cobweb_new/__init__.py

@@ -0,0 +1,2 @@
+from .launchers import LauncherAir, LauncherPro, LauncherApi
+from .constant import CrawlerModel

cobweb-launcher-1.2.27/cobweb_new/base/__init__.py

@@ -0,0 +1,72 @@
+import time
+from inspect import isgenerator
+from typing import Callable, Union
+
+from .common_queue import Queue
+from .response import Response
+from .request import Request
+from .item import BaseItem, ConsoleItem
+from .seed import Seed
+
+from .log import logger
+# from .decorators import decorator_oss_db, stop, pause
+import decorators
+
+
+class TaskQueue:
+
+    SEED = Queue()          # 添加任务种子队列
+    TODO = Queue()          # 任务种子队列
+    REQUEST = Queue()       # 请求队列
+
+    DOWNLOAD = Queue()      # 下载任务队列
+    RESPONSE = Queue()      # 响应队列
+    DONE = Queue()          # 下载完成队列
+
+    UPLOAD = Queue()        # 任务上传队列
+
+    DELETE = Queue()        # 任务删除队列
+
+    def __init__(self, db):
+        self.db = db
+
+    @staticmethod
+    def is_empty():
+        total_length = TaskQueue.SEED.length
+        total_length += TaskQueue.TODO.length
+        total_length += TaskQueue.REQUEST.length
+        total_length += TaskQueue.DOWNLOAD.length
+        total_length += TaskQueue.RESPONSE.length
+        total_length += TaskQueue.UPLOAD.length
+        total_length += TaskQueue.DONE.length
+        total_length += TaskQueue.DELETE.length
+        return not bool(total_length)
+    # @staticmethod
+    # def distribute(it):
+
+    @staticmethod
+    def process_task(it: Union[Seed, Request, Response, BaseItem], crawler_func: Callable):
+        try:
+            iterators = crawler_func(it)
+            if not isgenerator(iterators):
+                raise TypeError(f"{crawler_func.__name__} function isn't a generator")
+            for tk in iterators:
+                if isinstance(tk, Request):
+                    TaskQueue.DOWNLOAD.push(tk)
+                elif isinstance(tk, Response):
+                    TaskQueue.RESPONSE.push(tk)
+                elif isinstance(tk, BaseItem):
+                    TaskQueue.UPLOAD.push(tk)
+                elif isinstance(tk, Seed):
+                    TaskQueue.SEED.push(tk)
+        except Exception as e:
+            if not isinstance(it, BaseItem):
+                it.seed.params.retry += 1
+
+            time.sleep(5)
+
+
+class Distribute:
+    """
+    数据分发器，将数据分发到各个队列中
+    """

cobweb-launcher-1.2.27/cobweb_new/base/common_queue.py

@@ -0,0 +1,53 @@
+import time
+from collections import deque
+
+
+class Queue:
+
+    def __init__(self):
+        self._queue = deque()
+
+    @property
+    def length(self) -> int:
+        return len(self._queue)
+
+    def push(self, data, left: bool = False, direct_insertion: bool = False):
+        try:
+            if not data:
+                return None
+            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
+                self._queue.extendleft(data) if left else self._queue.extend(data)
+            else:
+                self._queue.appendleft(data) if left else self._queue.append(data)
+        except AttributeError:
+            pass
+
+    def pop(self, left: bool = True):
+        try:
+            return self._queue.popleft() if left else self._queue.pop()
+        except IndexError:
+            return None
+        except AttributeError:
+            return None
+
+    def clear(self):
+        self._queue.clear()
+
+    def get(self):
+        try:
+            yield self._queue.popleft()
+        except IndexError:
+            time.sleep(1)
+            yield None
+        except AttributeError:
+            yield None
+
+
+class RedisQueue(Queue):
+
+    def __init__(self, db):
+        super().__init__()
+        self.db = db
+
+    def pop(self, left: bool = True):
+        ...

cobweb-launcher-1.2.27/cobweb_new/base/decorators.py

@@ -0,0 +1,72 @@
+import time
+import threading
+from functools import wraps
+
+
+def add_thread(num=1):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(self, *args):
+            for i in range(num):
+                name = func.__name__ + "_" + str(i) if num > 1 else func.__name__
+                self._threads.append(threading.Thread(name=name, target=func, args=(self,) + args))
+        return wrapper
+
+    return decorator
+
+
+def pause(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        while not self.pause.is_set():
+            try:
+                func(self, *args, **kwargs)
+            except Exception as e:
+                pass
+                # logger.info(f"{func.__name__}: " + str(e))
+            finally:
+                time.sleep(0.1)
+
+    return wrapper
+
+
+def stop(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        while not self.stop.is_set():
+            try:
+                func(self, *args, **kwargs)
+            except Exception as e:
+                # logger.info(f"{func.__name__}: " + str(e))
+                pass
+            finally:
+                time.sleep(0.1)
+
+    return wrapper
+
+
+def decorator_oss_db(exception, retries=3):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(callback_func, *args, **kwargs):
+            result = None
+            for i in range(retries):
+                msg = None
+                try:
+                    return func(callback_func, *args, **kwargs)
+                except Exception as e:
+                    result = None
+                    msg = e
+                finally:
+                    if result:
+                        return result
+
+                    if i >= 2 and msg:
+                        raise exception(msg)
+
+        return wrapper
+
+    return decorator
+
+
+

cobweb-launcher-1.2.27/cobweb_new/base/item.py

@@ -0,0 +1,46 @@
+from .seed import Seed
+from collections import namedtuple
+
+
+class Item(type):
+
+    def __new__(cls, name, bases, dct):
+        new_class_instance = type.__new__(cls, name, bases, dct)
+        if name != "BaseItem":
+            table = getattr(new_class_instance, "__TABLE__")
+            fields = getattr(new_class_instance, "__FIELDS__")
+            new_class_instance.Data = namedtuple(table, fields)
+        return new_class_instance
+
+
+class BaseItem(metaclass=Item):
+
+    __TABLE__ = ""
+    __FIELDS__ = ""
+
+    def __init__(self, seed: Seed, **kwargs):
+        self.seed = seed
+
+        data = {}
+        for key, value in kwargs.items():
+            if key not in self.__FIELDS__:
+                self.__setattr__(key, value)
+            else:
+                data[key] = value
+
+        self.data = self.Data(**data)
+
+    @property
+    def to_dict(self):
+        return self.data._asdict()
+
+    @property
+    def table(self):
+        return self.Data.__name__
+
+
+class ConsoleItem(BaseItem):
+
+    __TABLE__ = "console"
+    __FIELDS__ = "data"
+

cobweb-launcher-1.2.27/cobweb_new/base/log.py

@@ -0,0 +1,94 @@
+import logging
+
+
+class ColorCodes:
+    # Text Reset
+    RESET = "\033[0m"
+
+    # Regular Colors
+    RED = "\033[31m"
+    GREEN = "\033[32m"
+    YELLOW = "\033[33m"
+    BLUE = "\033[34m"
+    PURPLE = "\033[35m"
+    CYAN = "\033[36m"
+    WHITE = "\033[37m"
+
+    # Bright Colors
+    BRIGHT_RED = "\033[91m"
+    BRIGHT_GREEN = "\033[92m"
+    BRIGHT_YELLOW = "\033[93m"
+    BRIGHT_BLUE = "\033[94m"
+    BRIGHT_PURPLE = "\033[95m"
+    BRIGHT_CYAN = "\033[96m"
+    BRIGHT_WHITE = "\033[97m"
+
+    # Background Colors
+    BG_RED = "\033[41m"
+    BG_GREEN = "\033[42m"
+    BG_YELLOW = "\033[43m"
+    BG_BLUE = "\033[44m"
+    BG_PURPLE = "\033[45m"
+    BG_CYAN = "\033[46m"
+    BG_WHITE = "\033[47m"
+
+    # Bright Background Colors
+    BG_BRIGHT_RED = "\033[101m"
+    BG_BRIGHT_GREEN = "\033[102m"
+    BG_BRIGHT_YELLOW = "\033[103m"
+    BG_BRIGHT_BLUE = "\033[104m"
+    BG_BRIGHT_PURPLE = "\033[105m"
+    BG_BRIGHT_CYAN = "\033[106m"
+    BG_BRIGHT_WHITE = "\033[107m"
+
+    # Text Styles
+    BOLD = "\033[1m"
+    DIM = "\033[2m"
+    ITALIC = "\033[3m"
+    UNDERLINE = "\033[4m"
+    BLINK = "\033[5m"
+    REVERSE = "\033[7m"
+    HIDDEN = "\033[8m"
+
+
+class Log:
+    logging.getLogger('oss2.api').setLevel(logging.WARNING)
+    logging.basicConfig(
+        level=logging.INFO,
+        format=f'%(asctime)s %(name)s [%(filename)s:%(lineno)d %(funcName)s]'
+               f' %(levelname)s -> %(message)s'
+    )
+    log = logging.getLogger()
+
+    def set_log_name(self, name):
+        self.__class__.log = logging.getLogger(name)
+
+    @property
+    def debug(self):
+        return self.__class__.log.debug
+
+    @property
+    def info(self):
+        return self.__class__.log.info
+
+    @property
+    def warning(self):
+        return self.__class__.log.warning
+
+    @property
+    def exception(self):
+        return self.__class__.log.exception
+
+    @property
+    def error(self):
+        return self.__class__.log.error
+
+    @property
+    def critical(self):
+        return self.__class__.log.critical
+
+
+logger = Log()
+
+
+