cobweb-launcher 1.1.23__py3-none-any.whl → 1.2.1__py3-none-any.whl

cobweb/__init__.py

@@ -1,2 +1,2 @@
-from .launchers import Launcher, LauncherPro
+from .launchers import LauncherAir, LauncherPro
 from .constant import CrawlerModel

cobweb/base/__init__.py

@@ -1,7 +1,7 @@
 from .common_queue import Queue
 from .response import Response
 from .request import Request
-from .item import BaseItem
+from .item import BaseItem, ConsoleItem
 from .seed import Seed
 
 from .log import logger

cobweb/base/item.py

@@ -37,3 +37,10 @@ class BaseItem(metaclass=Item):
     @property
     def table(self):
         return self.Data.__name__
+
+
+class ConsoleItem(BaseItem):
+
+    __TABLE__ = "console"
+    __FIELDS__ = "data"
+

cobweb/constant.py

@@ -30,6 +30,24 @@ class DealModel:
 
 class LogTemplate:
 
+    console_item = """
+    ----------------------- start - console pipeline -----------------
+            种子详情 \n{seed_detail}
+            解析详情 \n{parse_detail}
+    ----------------------- end  - console pipeline ------------------
+    """
+
+    launcher_air_polling = """
+    ----------------------- start - 轮训日志: {task} -----------------
+            内存队列 
+                种子数:  {doing_len}
+                待消费:  {todo_len}
+                已消费:  {done_len}
+            存储队列 
+                待上传:  {upload_len}
+    ----------------------- end  - 轮训日志: {task} ------------------
+    """
+
     launcher_pro_polling = """
 ----------------------- start - 轮训日志: {task} -----------------
         内存队列 
@@ -69,4 +87,8 @@ class LogTemplate:
         response 
             status        :    {status} \n{response}
 ------------------------------------------------------------------
-"""
+"""
+
+    @staticmethod
+    def log_info(item: dict) -> str:
+        return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

cobweb/crawlers/__init__.py

@@ -1,2 +1 @@
-from .base_crawler import Crawler
-from .file_crawler import FileCrawlerAir
+from .crawler import Crawler

cobweb/crawlers/crawler.py

@@ -0,0 +1,160 @@
+import json
+import threading
+import time
+import traceback
+from inspect import isgenerator
+from typing import Union, Callable, Mapping
+
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.base import (
+    Queue, 
+    Seed, 
+    BaseItem, 
+    Request, 
+    Response, 
+    ConsoleItem,
+    logger
+)
+
+
+class Crawler(threading.Thread):
+
+    def __init__(
+            self,
+            stop: threading.Event,
+            pause: threading.Event,
+            launcher_queue: Union[Mapping[str, Queue]],
+            custom_func: Union[Mapping[str, Callable]],
+            thread_num: int,
+            max_retries: int
+    ):
+        super().__init__()
+
+        self._stop = stop
+        self._pause = pause
+        self._new = launcher_queue["new"]
+        self._todo = launcher_queue["todo"]
+        self._done = launcher_queue["done"]
+        self._upload = launcher_queue["upload"]
+
+        for func_name, _callable in custom_func.items():
+            if isinstance(_callable, Callable):
+                self.__setattr__(func_name, _callable)
+
+        self.thread_num = thread_num
+        self.max_retries = max_retries
+
+    @staticmethod
+    def request(seed: Seed) -> Union[Request, BaseItem]:
+        yield Request(seed.url, seed, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> BaseItem:
+        upload_item = item.to_dict
+        upload_item["text"] = item.response.text
+        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
+
+    # def get_seed(self) -> Seed:
+    #     return self._todo.pop()
+
+    def distribute(self, item, seed):
+        if isinstance(item, BaseItem):
+            self._upload.push(item)
+        elif isinstance(item, Seed):
+            self._new.push(item)
+        elif isinstance(item, str) and item == DealModel.poll:
+            self._todo.push(seed)
+        elif isinstance(item, str) and item == DealModel.done:
+            self._done.push(seed)
+        elif isinstance(item, str) and item == DealModel.fail:
+            seed.params.seed_status = DealModel.fail
+            self._done.push(seed)
+        else:
+            raise TypeError("yield value type error!")
+
+    def spider(self):
+        while not self._stop.is_set():
+
+            seed = self._todo.pop()
+
+            if not seed:
+                time.sleep(1)
+                continue
+
+            elif seed.params.retry >= self.max_retries:
+                seed.params.seed_status = DealModel.fail
+                self._done.push(seed)
+                continue
+
+            seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
+
+            try:
+                request_iterators = self.request(seed)
+
+                if not isgenerator(request_iterators):
+                    raise TypeError("request function isn't a generator!")
+
+                iterator_status = False
+
+                for request_item in request_iterators:
+
+                    iterator_status = True
+
+                    if isinstance(request_item, Request):
+                        iterator_status = False
+                        download_iterators = self.download(request_item)
+                        if not isgenerator(download_iterators):
+                            raise TypeError("download function isn't a generator")
+
+                        for download_item in download_iterators:
+                            iterator_status = True
+                            if isinstance(download_item, Response):
+                                iterator_status = False
+                                logger.info(LogTemplate.download_info.format(
+                                    detail=seed_detail_log_info,
+                                    retry=seed.params.retry,
+                                    priority=seed.params.priority,
+                                    seed_version=seed.params.seed_version,
+                                    identifier=seed.identifier or "",
+                                    status=download_item.response,
+                                    response=LogTemplate.log_info(download_item.to_dict)
+                                ))
+                                parse_iterators = self.parse(download_item)
+                                if not isgenerator(parse_iterators):
+                                    raise TypeError("parse function isn't a generator")
+                                for parse_item in parse_iterators:
+                                    iterator_status = True
+                                    if isinstance(parse_item, Response):
+                                        raise TypeError("upload_item can't be a Response instance")
+                                    self.distribute(parse_item, seed)
+                            else:
+                                self.distribute(download_item, seed)
+                    else:
+                        self.distribute(request_item, seed)
+
+                if not iterator_status:
+                    raise ValueError("request/download/parse function yield value error!")
+            except Exception as e:
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info,
+                    retry=seed.params.retry,
+                    priority=seed.params.priority,
+                    seed_version=seed.params.seed_version,
+                    identifier=seed.identifier or "",
+                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                ))
+                seed.params.retry += 1
+                self._todo.push(seed)
+            finally:
+                time.sleep(0.1)
+        logger.info("spider thread close")
+
+    def run(self):
+        for index in range(self.thread_num):
+            threading.Thread(name=f"spider_{index}", target=self.spider).start()
+

cobweb/launchers/__init__.py

@@ -1,2 +1,2 @@
-from .launcher import Launcher
+from .launcher_air import LauncherAir
 from .launcher_pro import LauncherPro

cobweb/launchers/launcher.py

@@ -15,15 +15,16 @@ class Launcher(threading.Thread):
     __DOING__ = {}
 
     __CUSTOM_FUNC__ = {
-        "download": None,
-        "download_midware": None,
-        "parse": None,
+        # "download": None,
+        # "request": None,
+        # "parse": None,
     }
 
     __LAUNCHER_QUEUE__ = {
         "new": Queue(),
         "todo": Queue(),
         "done": Queue(),
+        "upload": Queue()
     }
 
     __LAUNCHER_FUNC__ = [
@@ -76,9 +77,13 @@ class Launcher(threading.Thread):
         self._done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
         self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
 
+        self._spider_thread_num = setting.SPIDER_MAX_RETRIES
+        self._spider_max_retries = setting.SPIDER_THREAD_NUM
+
         self._done_model = setting.DONE_MODEL
+        self._task_model = setting.TASK_MODEL
 
-        self._upload_queue = Queue()
+        # self._upload_queue = Queue()
 
     @property
     def start_seeds(self):
@@ -120,8 +125,8 @@ class Launcher(threading.Thread):
         自定义parse函数, xxxItem为自定义的存储数据类型
         use case:
             from cobweb.base import Request, Response
-            @launcher.download
-            def download(item: Response) -> BaseItem:
+            @launcher.parse
+            def parse(item: Response) -> BaseItem:
                ...
                yield xxxItem(seed, **kwargs)
         """
@@ -133,6 +138,33 @@ class Launcher(threading.Thread):
         for seed in seeds:
             self.__DOING__.pop(seed, None)
 
+    def _execute(self):
+        for func_name in self.__LAUNCHER_FUNC__:
+            threading.Thread(name=func_name, target=getattr(self, func_name)).start()
+            time.sleep(2)
+
+    def run(self):
+        threading.Thread(target=self._execute_heartbeat).start()
+
+        self._Crawler(
+            stop=self._stop, pause=self._pause,
+            launcher_queue=self.__LAUNCHER_QUEUE__,
+            custom_func=self.__CUSTOM_FUNC__,
+            thread_num = self._spider_thread_num,
+            max_retries = self._spider_max_retries
+        ).start()
+
+        self._Pipeline(
+            stop=self._stop, pause=self._pause,
+            upload=self.__LAUNCHER_QUEUE__["upload"],
+            done=self.__LAUNCHER_QUEUE__["done"],
+            upload_size=self._upload_queue_max_size,
+            wait_seconds=self._upload_queue_wait_seconds
+        ).start()
+
+        self._execute()
+        self._polling()
+
     def _execute_heartbeat(self):
         pass
 
@@ -151,52 +183,6 @@ class Launcher(threading.Thread):
     def _delete(self):
         pass
 
-    def _execute(self):
-        for func_name in self.__LAUNCHER_FUNC__:
-            threading.Thread(name=func_name, target=getattr(self, func_name)).start()
-            time.sleep(2)
-
     def _polling(self):
+        pass
 
-        check_emtpy_times = 0
-
-        while not self._stop.is_set():
-
-            queue_not_empty_count = 0
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-
-            if self._pause.is_set() and queue_not_empty_count != 0:
-                self._pause.clear()
-                self._execute()
-
-            elif queue_not_empty_count == 0:
-                check_emtpy_times += 1
-            else:
-                check_emtpy_times = 0
-
-            if check_emtpy_times > 2:
-                check_emtpy_times = 0
-                self.__DOING__ = {}
-                self._pause.set()
-
-    def run(self):
-        threading.Thread(target=self._execute_heartbeat).start()
-
-        self._Crawler(
-            upload_queue=self._upload_queue,
-            custom_func=self.__CUSTOM_FUNC__,
-            launcher_queue=self.__LAUNCHER_QUEUE__,
-        ).start()
-
-        self._Pipeline(
-            upload_queue=self._upload_queue,
-            done_queue=self.__LAUNCHER_QUEUE__["done"],
-            upload_queue_size=self._upload_queue_max_size,
-            upload_wait_seconds=self._upload_queue_wait_seconds
-        ).start()
-
-        self._execute()
-        self._polling()

cobweb/launchers/launcher_air.py

@@ -0,0 +1,88 @@
+import time
+
+from cobweb.constant import LogTemplate
+from cobweb.base import logger
+from .launcher import Launcher
+
+
+class LauncherAir(Launcher):
+
+    def _scheduler(self):
+        if self.start_seeds:
+            self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
+
+    def _insert(self):
+        while not self._pause.is_set():
+            seeds = {}
+            status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+            for _ in range(self._new_queue_max_size):
+                seed = self.__LAUNCHER_QUEUE__['new'].pop()
+                if not seed:
+                    break
+                seeds[seed.to_string] = seed.params.priority
+            if seeds:
+                self.__LAUNCHER_QUEUE__['todo'].push(seeds)
+            if status:
+                time.sleep(self._new_queue_wait_seconds)
+
+    def _delete(self):
+        while not self._pause.is_set():
+            seeds = []
+            status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+
+            for _ in range(self._done_queue_max_size):
+                seed = self.__LAUNCHER_QUEUE__['done'].pop()
+                if not seed:
+                    break
+                seeds.append(seed.to_string)
+
+            if seeds:
+                self._remove_doing_seeds(seeds)
+
+            if status:
+                time.sleep(self._done_queue_wait_seconds)
+
+    def _polling(self):
+
+        check_emtpy_times = 0
+
+        while not self._stop.is_set():
+
+            queue_not_empty_count = 0
+            pooling_wait_seconds = 30
+
+            for q in self.__LAUNCHER_QUEUE__.values():
+                if q.length != 0:
+                    queue_not_empty_count += 1
+
+            if queue_not_empty_count == 0:
+                pooling_wait_seconds = 3
+                if self._pause.is_set():
+                    check_emtpy_times = 0
+                    if not self._task_model:
+                        logger.info("Done! Ready to close thread...")
+                        self._stop.set()
+                elif check_emtpy_times > 2:
+                    self.__DOING__ = {}
+                    self._pause.set()
+                else:
+                    logger.info(
+                        "check whether the task is complete, "
+                        f"reset times {3 - check_emtpy_times}"
+                    )
+                    check_emtpy_times += 1
+            elif self._pause.is_set():
+                self._pause.clear()
+                self._execute()
+            else:
+                logger.info(LogTemplate.launcher_air_polling.format(
+                    task=self.task,
+                    doing_len=len(self.__DOING__.keys()),
+                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
+                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
+                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
+                ))
+
+            time.sleep(pooling_wait_seconds)
+
+

cobweb/launchers/launcher_pro.py

@@ -3,19 +3,19 @@ import threading
 
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
-from cobweb.launchers import Launcher
 from cobweb.constant import DealModel, LogTemplate
+from .launcher import Launcher
 
 
 class LauncherPro(Launcher):
 
     def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__(task, project, custom_setting, **kwargs)
-        self._todo = "{%s:%s}:todo" % (project, task)
-        self._done = "{%s:%s}:done" % (project, task)
-        self._fail = "{%s:%s}:fail" % (project, task)
-        self._heartbeat = "heartbeat:%s_%s" % (project, task)
-        self._reset_lock = "lock:reset:%s_%s" % (project, task)
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._done_key = "{%s:%s}:done" % (project, task)
+        self._fail_key = "{%s:%s}:fail" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
         self._db = RedisDB()
 
         self._heartbeat_start_event = threading.Event()
@@ -23,12 +23,12 @@ class LauncherPro(Launcher):
 
     @property
     def heartbeat(self):
-        return self._db.exists(self._heartbeat)
+        return self._db.exists(self._heartbeat_key)
 
     def _execute_heartbeat(self):
         while not self._stop.is_set():
             if self._heartbeat_start_event.is_set():
-                self._db.setex(self._heartbeat, 3)
+                self._db.setex(self._heartbeat_key, 3)
             time.sleep(2)
 
     def _reset(self):
@@ -39,15 +39,15 @@ class LauncherPro(Launcher):
         while not self._pause.is_set():
             reset_wait_seconds = 30
             start_reset_time = int(time.time())
-            if self._db.lock(self._reset_lock, t=120):
+            if self._db.lock(self._reset_lock_key, t=120):
                 if not self.heartbeat:
                     self._heartbeat_start_event.set()
 
                 _min = -int(time.time()) + self._seed_reset_seconds \
                     if self.heartbeat or not init else "-inf"
 
-                self._db.members(self._todo, 0, _min=_min, _max="(0")
-                self._db.delete(self._reset_lock)
+                self._db.members(self._todo_key, 0, _min=_min, _max="(0")
+                self._db.delete(self._reset_lock_key)
 
                 ttl = 120 - int(time.time()) + start_reset_time
                 reset_wait_seconds = max(ttl, 1)
@@ -61,14 +61,14 @@ class LauncherPro(Launcher):
         if self.start_seeds:
             self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
         while not self._pause.is_set():
-            if not self._db.zcount(self._todo, 0, "(1000"):
+            if not self._db.zcount(self._todo_key, 0, "(1000"):
                 time.sleep(self._scheduler_wait_seconds)
                 continue
             if self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
                 time.sleep(self._todo_queue_full_wait_seconds)
                 continue
             members = self._db.members(
-                self._todo, int(time.time()),
+                self._todo_key, int(time.time()),
                 count=self._todo_queue_size,
                 _min=0, _max="(1000"
             )
@@ -90,7 +90,7 @@ class LauncherPro(Launcher):
                     break
                 seeds[seed.to_string] = seed.params.priority
             if seeds:
-                self._db.zadd(self._todo, seeds, nx=True)
+                self._db.zadd(self._todo_key, seeds, nx=True)
             if status:
                 time.sleep(self._new_queue_wait_seconds)
 
@@ -102,7 +102,7 @@ class LauncherPro(Launcher):
             if self.__DOING__:
                 refresh_time = int(time.time())
                 seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
-                self._db.zadd(self._todo, item=seeds, xx=True)
+                self._db.zadd(self._todo_key, item=seeds, xx=True)
             time.sleep(30)
 
     def _delete(self):
@@ -124,13 +124,13 @@ class LauncherPro(Launcher):
                 else:
                     seeds.append(seed.to_string)
             if seeds:
-                self._db.zrem(self._todo, *seeds)
+                self._db.zrem(self._todo_key, *seeds)
                 self._remove_doing_seeds(seeds)
             if s_seeds:
-                self._db.done([self._todo, self._done], *s_seeds)
+                self._db.done([self._todo_key, self._done_key], *s_seeds)
                 self._remove_doing_seeds(s_seeds)
             if f_seeds:
-                self._db.done([self._todo, self._fail], *f_seeds)
+                self._db.done([self._todo_key, self._fail_key], *f_seeds)
                 self._remove_doing_seeds(f_seeds)
 
             if status:
@@ -141,32 +141,79 @@ class LauncherPro(Launcher):
         while not self._stop.is_set():
             queue_not_empty_count = 0
             pooling_wait_seconds = 30
+
             for q in self.__LAUNCHER_QUEUE__.values():
                 if q.length != 0:
                     queue_not_empty_count += 1
-            if self._pause.is_set():
-                self._pause.clear()
-                self._execute()
-            elif queue_not_empty_count == 0:
-                pooling_wait_seconds = 5
-                check_emtpy_times += 1
-            else:
-                check_emtpy_times = 0
 
-            if not self._db.zcount(self._todo, _min=0, _max="(1000") and check_emtpy_times > 2:
-                check_emtpy_times = 0
-                self.__DOING__ = {}
-                self._pause.set()
-
-            if not self._pause.is_set():
+            if queue_not_empty_count == 0:
+                pooling_wait_seconds = 3
+                if self._pause.is_set():
+                    check_emtpy_times = 0
+                    if not self._task_model:
+                        logger.info("Done! ready to close thread...")
+                        self._stop.set()
+                    elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
+                        logger.info(f"Recovery {self.task} task run！")
+                        self._pause.clear()
+                        self._execute()
+                    else:
+                        logger.info("pause! waiting for resume...")
+                elif check_emtpy_times > 2:
+                    self.__DOING__ = {}
+                    self._pause.set()
+                else:
+                    logger.info(
+                        "check whether the task is complete, "
+                        f"reset times {3 - check_emtpy_times}"
+                    )
+                    check_emtpy_times += 1
+            # elif self._pause.is_set():
+            #     self._pause.clear()
+            #     self._execute()
+            else:
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
                     doing_len=len(self.__DOING__.keys()),
                     todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
                     done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    redis_seed_count=self._db.zcount(self._todo, "-inf", "+inf"),
-                    redis_todo_len=self._db.zcount(self._todo, 0, "(1000"),
-                    redis_doing_len=self._db.zcount(self._todo, "-inf", "(0"),
-                    upload_len=self._upload_queue.length
+                    redis_seed_count=self._db.zcount(self._todo_key, "-inf", "+inf"),
+                    redis_todo_len=self._db.zcount(self._todo_key, 0, "(1000"),
+                    redis_doing_len=self._db.zcount(self._todo_key, "-inf", "(0"),
+                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
                 ))
+
             time.sleep(pooling_wait_seconds)
+            # if self._pause.is_set():
+            #     self._pause.clear()
+            #     self._execute()
+            #
+            # elif queue_not_empty_count == 0:
+            #     pooling_wait_seconds = 5
+            #     check_emtpy_times += 1
+            # else:
+            #     check_emtpy_times = 0
+            #
+            # if not self._db.zcount(self._todo, _min=0, _max="(1000") and check_emtpy_times > 2:
+            #     check_emtpy_times = 0
+            #     self.__DOING__ = {}
+            #     self._pause.set()
+            #
+            # time.sleep(pooling_wait_seconds)
+            #
+            # if not self._pause.is_set():
+            #     logger.info(LogTemplate.launcher_pro_polling.format(
+            #         task=self.task,
+            #         doing_len=len(self.__DOING__.keys()),
+            #         todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
+            #         done_len=self.__LAUNCHER_QUEUE__['done'].length,
+            #         redis_seed_count=self._db.zcount(self._todo, "-inf", "+inf"),
+            #         redis_todo_len=self._db.zcount(self._todo, 0, "(1000"),
+            #         redis_doing_len=self._db.zcount(self._todo, "-inf", "(0"),
+            #         upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
+            #     ))
+            # elif not self._task_model:
+            #     self._stop.set()
+
+        logger.info("Done! Ready to close thread...")
+

cobweb/pipelines/__init__.py

@@ -1,2 +1,3 @@
-from .base_pipeline import Pipeline
-from .loghub_pipeline import LoghubPipeline
+from .pipeline import Pipeline
+from .pipeline_console import Console
+from .pipeline_loghub import Loghub

cobweb/pipelines/pipeline.py

@@ -0,0 +1,60 @@
+import time
+import threading
+
+from abc import ABC, abstractmethod
+from cobweb.base import BaseItem, Queue, logger
+
+
+class Pipeline(threading.Thread, ABC):
+
+    def __init__(
+            self,
+            stop: threading.Event,
+            pause: threading.Event,
+            upload: Queue, done: Queue,
+            upload_size: int,
+            wait_seconds: int
+    ):
+        super().__init__()
+        self._stop = stop
+        self._pause = pause
+        self._upload = upload
+        self._done = done
+
+        self.upload_size = upload_size
+        self.wait_seconds = wait_seconds
+
+    @abstractmethod
+    def build(self, item: BaseItem) -> dict:
+        pass
+
+    @abstractmethod
+    def upload(self, table: str, data: list) -> bool:
+        pass
+
+    def run(self):
+        while not self._stop.is_set():
+            status = self._upload.length < self.upload_size
+            if status:
+                time.sleep(self.wait_seconds)
+            data_info, seeds = {}, []
+            for _ in range(self.upload_size):
+                item = self._upload.pop()
+                if not item:
+                    break
+                data = self.build(item)
+                seeds.append(item.seed)
+                data_info.setdefault(item.table, []).append(data)
+            for table, datas in data_info.items():
+                try:
+                    self.upload(table, datas)
+                    status = True
+                except Exception as e:
+                    logger.info(e)
+                    status = False
+                if status:
+                    self._done.push(seeds)
+
+        logger.info("upload pipeline close!")
+
+

cobweb/pipelines/pipeline_console.py

@@ -0,0 +1,22 @@
+from cobweb.base import ConsoleItem, logger
+from cobweb.constant import LogTemplate
+from cobweb.pipelines import Pipeline
+
+
+class Console(Pipeline):
+
+    def build(self, item: ConsoleItem):
+        return {
+            "seed": item.seed.to_dict,
+            "data": item.to_dict
+        }
+
+    def upload(self, table, datas):
+        for data in datas:
+            parse_detail = LogTemplate.log_info(data["data"])
+            if len(parse_detail) > 500:
+                parse_detail = parse_detail[:500] + " ...\n" + " " * 12 + "-- Text is too long and details are omitted!"
+            logger.info(LogTemplate.console_item.format(
+                seed_detail=LogTemplate.log_info(data["seed"]),
+                parse_detail=parse_detail
+            ))

cobweb/pipelines/pipeline_loghub.py

@@ -0,0 +1,34 @@
+import json
+
+from cobweb import setting
+from cobweb.base import BaseItem
+from cobweb.pipelines import Pipeline
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+
+class Loghub(Pipeline):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = LogClient(**setting.LOGHUB_CONFIG)
+
+    def build(self, item: BaseItem):
+        log_item = LogItem()
+        temp = item.to_dict
+        for key, value in temp.items():
+            if not isinstance(value, str):
+                temp[key] = json.dumps(value, ensure_ascii=False)
+        contents = sorted(temp.items())
+        log_item.set_contents(contents)
+        return log_item
+
+    def upload(self, table, datas):
+        request = PutLogsRequest(
+            project=setting.LOGHUB_PROJECT,
+            logstore=table,
+            topic=setting.LOGHUB_TOPIC,
+            source=setting.LOGHUB_SOURCE,
+            logitems=datas,
+            compress=True
+        )
+        self.client.put_logs(request=request)

cobweb/setting.py

@@ -30,8 +30,8 @@ OSS_MIN_UPLOAD_SIZE = 1024
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
 
-# 数据上传链路
-PIPELINE = "cobweb.pipelines.loghub_pipeline.LoghubPipeline"
+# 数据存储链路
+PIPELINE = "cobweb.pipelines.pipeline_console.Console"
 
 
 # Launcher 等待时间
@@ -52,12 +52,12 @@ UPLOAD_QUEUE_MAX_SIZE = 100  # upload队列长度
 # DONE_MODEL IN (0, 1), 种子完成模式
 DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加至失败队列；1:种子消费成功添加至成功队列，失败添加至失败队列
 
-# DOWNLOAD_MODEL IN (0, 1), 下载模式
-DOWNLOAD_MODEL = 0  # 0: 通用下载；1:文件下载
-
 # spider
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
 
+# 任务模式
+TASK_MODEL = 0  # 0:单次，1:常驻
+
 # 文件下载响应类型过滤
-FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]
+# FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]

cobweb/utils/tools.py

@@ -38,5 +38,5 @@ def dynamic_load_class(model_info):
     raise TypeError()
 
 
-def download_log_info(item:dict) -> str:
-    return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])
+# def download_log_info(item:dict) -> str:
+#     return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

cobweb_launcher-1.2.1.dist-info/METADATA

@@ -0,0 +1,204 @@
+Metadata-Version: 2.1
+Name: cobweb-launcher
+Version: 1.2.1
+Summary: spider_hole
+Home-page: https://github.com/Juannie-PP/cobweb
+Author: Juannie-PP
+Author-email: 2604868278@qq.com
+License: MIT
+Keywords: cobweb-launcher, cobweb
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests (>=2.19.1)
+Requires-Dist: oss2 (>=2.18.1)
+Requires-Dist: redis (>=4.4.4)
+Requires-Dist: aliyun-log-python-sdk
+
+# cobweb 
+cobweb是一个基于python的分布式爬虫调度框架，目前支持分布式爬虫，单机爬虫，支持自定义数据库，支持自定义数据存储，支持自定义数据处理等操作。  
+
+cobweb主要由3个模块和一个配置文件组成：Launcher启动器、Crawler采集器、Pipeline存储和setting配置文件。
+1. Launcher启动器：用于启动爬虫任务，控制爬虫任务的执行流程，以及数据存储和数据处理。
+框架提供两种启动器模式：LauncherAir、LauncherPro，分别对应单机爬虫模式和分布式调度模式。
+2. Crawler采集器：用于控制采集流程、数据下载和数据处理。
+框架提供了基础的采集器，用于控制采集流程、数据下载和数据处理，用户也可在创建任务时自定义请求、下载和解析方法，具体看使用方法介绍。
+3. Pipeline存储：用于存储采集到的数据，支持自定义数据存储和数据处理。框架提供了Console和Loghub两种存储方式，用户也可继承Pipeline抽象类自定义存储方式。
+4. setting配置文件：用于配置采集器、存储器、队列长度、采集线程数等参数，框架提供了默认配置，用户也可自定义配置。
+## 安装
+```
+pip3 install --upgrade cobweb-launcher
+```
+## 使用方法介绍
+### 1. 任务创建
+- LauncherAir任务创建
+```python
+from cobweb import LauncherAir
+
+# 创建启动器
+app = LauncherAir(task="test", project="test")
+
+# 设置采集种子
+app.SEEDS = [{
+    "url": "https://www.baidu.com"
+}]
+...
+# 启动任务
+app.start()
+```
+- LauncherPro任务创建  
+LauncherPro依赖redis实现分布式调度，使用LauncherPro启动器需要完成环境变量的配置或自定义setting文件中的redis配置，如何配置查看`2. 自定义配置文件参数`
+```python
+from cobweb import LauncherPro
+
+# 创建启动器
+app = LauncherPro(
+    task="test",
+    project="test"
+)
+...
+# 启动任务
+app.start()
+```
+### 2. 自定义配置文件参数
+- 通过自定义setting文件，配置文件导入字符串方式  
+> 默认配置文件：import cobweb.setting  
+> 不推荐！！！目前有bug，随缘使用...
+例如：同级目录下自定义创建了setting.py文件。
+```python
+from cobweb import LauncherAir
+
+app = LauncherAir(
+    task="test", 
+    project="test",
+    setting="import setting"
+)
+
+...
+
+app.start()
+```
+- 自定义修改setting中对象值
+```python
+from cobweb import LauncherPro
+
+# 创建启动器
+app = LauncherPro(
+    task="test",
+    project="test",
+    REDIS_CONFIG = {
+        "host": ...,
+        "password":...,
+        "port": ...,
+        "db": ...
+    }
+)
+...
+# 启动任务
+app.start()
+```
+### 3. 自定义请求
+`@app.request`使用装饰器封装自定义请求方法，作用于发生请求前的操作，返回Request对象或继承于BaseItem对象，用于控制请求参数。
+```python
+from typing import Union
+from cobweb import LauncherAir
+from cobweb.base import Seed, Request, BaseItem
+
+app = LauncherAir(
+    task="test", 
+    project="test"
+)
+
+...
+
+@app.request
+def request(seed: Seed) -> Union[Request, BaseItem]:
+    # 可自定义headers，代理，构造请求参数等操作
+    proxies = {"http": ..., "https": ...}
+    yield Request(seed.url, seed, ..., proxies=proxies, timeout=15)
+    # yield xxxItem(seed, ...) # 跳过请求和解析直接进入数据存储流程
+    
+...
+
+app.start()
+```
+> 默认请求方法  
+> def request(seed: Seed) -> Union[Request, BaseItem]:  
+>     yield Request(seed.url, seed, timeout=5)
+### 4. 自定义下载
+`@app.download`使用装饰器封装自定义下载方法，作用于发生请求时的操作，返回Response对象或继承于BaseItem对象，用于控制请求参数。
+```python
+from typing import Union
+from cobweb import LauncherAir
+from cobweb.base import Request, Response, BaseItem
+
+app = LauncherAir(
+    task="test", 
+    project="test"
+)
+
+...
+
+@app.download
+def download(item: Request) -> Union[BaseItem, Response]:
+    ...
+    response = ...
+    ...
+    yield Response(item.seed, response, ...) # 返回Response对象，进行解析
+    # yield xxxItem(seed, ...) # 跳过请求和解析直接进入数据存储流程
+    
+...
+
+app.start()
+```
+> 默认下载方法  
+> def download(item: Request) -> Union[Seed, BaseItem, Response, str]:  
+>     response = item.download()  
+>     yield Response(item.seed, response, **item.to_dict)
+### 5. 自定义解析
+自定义解析需要由一个存储数据类和解析方法组成。存储数据类继承于BaseItem的对象，规定存储表名及字段，
+解析方法返回继承于BaseItem的对象，yield返回进行控制数据存储流程。
+```python
+from typing import Union
+from cobweb import LauncherAir
+from cobweb.base import Seed, Response, BaseItem
+
+class TestItem(BaseItem):
+    __TABLE__ = "test_data" # 表名
+    __FIELDS__ = "field1, field2, field3" # 字段名
+
+app = LauncherAir(
+    task="test", 
+    project="test"
+)
+
+...
+
+@app.parse
+def parse(item: Response) -> Union[Seed, BaseItem]:
+    ...
+    yield TestItem(item.seed, field1=..., field2=..., field3=...)
+    # yield Seed(...) # 构造新种子推送至消费队列
+    
+...
+
+app.start()
+```
+> 默认解析方法  
+> def parse(item: Request) -> Union[Seed, BaseItem]:  
+>     upload_item = item.to_dict  
+>     upload_item["text"] = item.response.text  
+>     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
+## need deal
+- 队列优化完善，使用queue的机制wait()同步各模块执行？
+- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- 去重过滤（布隆过滤器等）
+- 单机防丢失
+- excel、mysql、redis数据完善
+
+> 未更新流程图！！！
+![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
+
+

cobweb_launcher-1.2.1.dist-info/RECORD

@@ -0,0 +1,37 @@
+cobweb/__init__.py,sha256=uMHyf4Fekbyw2xBCbkA8R0LwCpBJf5p_7pWbh60ZWYk,83
+cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
+cobweb/setting.py,sha256=zOO1cA_zQd4Q0CzY_tdSfdo-10L4QIVpm4382wbP5BQ,1906
+cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
+cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
+cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
+cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
+cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
+cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
+cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
+cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
+cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
+cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
+cobweb/crawlers/crawler.py,sha256=_4bU4rlg-oQTztWibtAEKY4nzFxpZ-FBu6KuwhEkewE,6003
+cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
+cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
+cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
+cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
+cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
+cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
+cobweb/launchers/launcher.py,sha256=_oPoFNXp3Rrf5eK2tcHtwHZi2pKbuNPBf_VRvj9dJUc,5379
+cobweb/launchers/launcher_air.py,sha256=zHVEJqQCxYU1WDnqQzzEHbEXasR1GmKevujQkCfFt5o,2947
+cobweb/launchers/launcher_pro.py,sha256=046x07CKK26qc69hOAoDPNVkAXAiFwmR0hz2EsDh_tc,8859
+cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
+cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
+cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
+cobweb/pipelines/pipeline.py,sha256=29O7CzASDv52NJl_1KWlKYxNd79HiaQAm0oO5K9PQzw,1650
+cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
+cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
+cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
+cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
+cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
+cobweb_launcher-1.2.1.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.1.dist-info/METADATA,sha256=M-72J_GDC29Vz2KGFB9nZCqHlK8YCYhVioR_qwOnjrs,6489
+cobweb_launcher-1.2.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.1.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.1.dist-info/RECORD,,

cobweb_launcher-1.1.23.dist-info/METADATA

@@ -1,48 +0,0 @@
-Metadata-Version: 2.1
-Name: cobweb-launcher
-Version: 1.1.23
-Summary: spider_hole
-Home-page: https://github.com/Juannie-PP/cobweb
-Author: Juannie-PP
-Author-email: 2604868278@qq.com
-License: MIT
-Keywords: cobweb-launcher, cobweb
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: requests (>=2.19.1)
-Requires-Dist: oss2 (>=2.18.1)
-Requires-Dist: redis (>=4.4.4)
-Requires-Dist: aliyun-log-python-sdk
-
-# cobweb
-
-> 通用爬虫框架： 1.单机模式采集框架；2.分布式采集框架
-> 
->  5部分
->  
-> 1. starter -- 启动器
-> 
-> 2. scheduler -- 调度器
-> 
-> 3. distributor -- 分发器
-> 
-> 4. storer -- 存储器
-> 
-> 5. utils -- 工具函数
-> 
-
-need deal
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 防丢失（单机模式可以通过日志文件进行检查种子）
-- 自定义数据库的功能
-- excel、mysql、redis数据完善
-
-
-![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
-
-

cobweb_launcher-1.1.23.dist-info/RECORD

@@ -1,32 +0,0 @@
-cobweb/__init__.py,sha256=1vB4yfJvrBVnixRRVNHLrJm8L0l5p_zOzLol5vKwP9M,80
-cobweb/constant.py,sha256=dMzvtsPje_5VgtMoOn-EvDtgycUWTKBKmDzX-CDfHDY,2025
-cobweb/setting.py,sha256=bg_oO85xsxwKqNUJScoVUI2v4e2lzSNOpsoW0tIaOrg,1953
-cobweb/base/__init__.py,sha256=diiK5MygQaWjlWNLbW6eUIg-93O6glMGC9WLNM5jyOc,209
-cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
-cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
-cobweb/base/item.py,sha256=pMriHStzUXtSvIf5Z3KXsP-bCvjlG1gM3z33wWeuoH8,966
-cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
-cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
-cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
-cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
-cobweb/crawlers/__init__.py,sha256=Rr3DTjD-abMA1_FYcQJZYNvQvcWMuEVcsIU6duqHrw4,75
-cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
-cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
-cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
-cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
-cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
-cobweb/launchers/launcher.py,sha256=LQdlpaF4fafEX01_3B9CB6hD-6YMPag6QOGl9rDprNE,5707
-cobweb/launchers/launcher_pro.py,sha256=VH_HnSQGAzxod6aFB3ZySi6-5Yp4vI-VE3u1XC1dfWg,6697
-cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
-cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
-cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
-cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
-cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
-cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.23.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.23.dist-info/METADATA,sha256=X93ltTYRGmvdT8oWACxycFetJ2KV3BgZdBP_MtDkE5U,1246
-cobweb_launcher-1.1.23.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.23.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.23.dist-info/RECORD,,