cobweb-launcher 3.1.11__tar.gz → 3.1.12__tar.gz

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.11
+Version: 3.1.12
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/base/__init__.py

@@ -1,9 +1,7 @@
+from .item import BaseItem, CSVItem
 from .common_queue import Queue
 from .response import Response
 from .request import Request
-from .item import BaseItem, ConsoleItem
+from .logger import logger
 from .seed import Seed
 
-from .log import logger
-# from .decorators import decorator_oss_db
-

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/base/item.py

@@ -12,6 +12,15 @@ class Item(type):
             new_class_instance.Data = namedtuple(table, fields)
         return new_class_instance
 
+    def __getattr__(self, name):
+        return None
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
 
 class BaseItem(metaclass=Item):
 
@@ -38,9 +47,14 @@ class BaseItem(metaclass=Item):
     def table(self):
         return self.Data.__name__
 
+    @property
+    def fields(self):
+        return self.__FIELDS__
+
+
 
-class ConsoleItem(BaseItem):
+class CSVItem(BaseItem):
 
-    __TABLE__ = "console"
+    __TABLE__ = "cobweb"
     __FIELDS__ = "data"
 

cobweb-launcher-3.1.11/cobweb/base/log.py → cobweb-launcher-3.1.12/cobweb/base/logger.py

@@ -51,8 +51,8 @@ class ColorCodes:
     HIDDEN = "\033[8m"
 
 
-class Log:
-    logging.getLogger('oss2.api').setLevel(logging.WARNING)
+class Logger:
+    # logging.getLogger('oss2.api').setLevel(logging.WARNING)
     logging.basicConfig(
         level=logging.INFO,
         format=f'%(asctime)s %(name)s [%(filename)s:%(lineno)d %(funcName)s]'
@@ -88,7 +88,7 @@ class Log:
         return self.__class__.log.critical
 
 
-logger = Log()
+logger = Logger()
 
 
 

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/base/request.py

@@ -58,7 +58,7 @@ class Request:
                       f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
         return user_agent
 
-    def _build_header(self) -> dict:
+    def _build_header(self):
         if not self.request_setting.get("headers"):
             self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
         elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/base/response.py

@@ -17,6 +17,7 @@ class Response:
     @property
     def to_dict(self):
         _dict = self.__dict__.copy()
+        _dict.update(self.seed.to_dict)
         _dict.pop('seed')
         _dict.pop('response')
         return _dict

cobweb-launcher-3.1.12/cobweb/crawlers/crawler.py

@@ -0,0 +1,27 @@
+from typing import Any, Generator
+from cobweb.base import (
+    Seed,
+    BaseItem,
+    Request,
+    Response,
+    CSVItem,
+)
+
+
+class Crawler:
+
+    @staticmethod
+    def request(seed: Seed) -> Generator[Request, Response, None]:
+        yield Request(seed.url, seed, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Generator[Response, Any, None]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> Generator[BaseItem, Any, None]:
+        upload_item = item.to_dict
+        upload_item["content"] = getattr(item.response, "text", item.response)
+        yield CSVItem(item.seed, data=upload_item)
+

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/db/redis_db.py

@@ -1,16 +1,21 @@
-import redis
+import os
+
 import time
-from cobweb import setting
+import redis
 from redis.exceptions import ConnectionError, TimeoutError
 
 
 class RedisDB:
-    def __init__(self, **kwargs):
-        redis_config = kwargs or setting.REDIS_CONFIG
-        self.host = redis_config['host']
-        self.password = redis_config['password']
-        self.port = redis_config['port']
-        self.db = redis_config['db']
+    def __init__(
+            self,
+            host=None,
+            password=None,
+            port=6379, db=0
+    ):
+        self.host = host or os.getenv("REDIS_HOST", "localhost")
+        self.password = password or os.getenv("REDIS_PASSWORD")
+        self.port = port or os.getenv("REDIS_PORT", 6379)
+        self.db = db or os.getenv("REDIS_DB", 0)
 
         self.max_retries = 5
         self.retry_delay = 5
@@ -18,7 +23,6 @@ class RedisDB:
         self.connect()
 
     def connect(self):
-        """尝试连接 Redis"""
         retries = 0
         while retries < self.max_retries:
             try:
@@ -27,10 +31,9 @@ class RedisDB:
                     port=self.port,
                     password=self.password,
                     db=self.db,
-                    socket_timeout=5,  # 设置连接超时时间
-                    socket_connect_timeout=5  # 设置连接超时时间
+                    socket_timeout=5,
+                    socket_connect_timeout=5
                 )
-                # 测试连接是否成功
                 self.client.ping()
                 return
             except (ConnectionError, TimeoutError) as e:
@@ -205,7 +208,7 @@ class RedisDB:
         members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
         return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
 
-    def done(self, keys: list, *args) -> list:
+    def done(self, keys: list, *args):
         lua_script = """
         for i, member in ipairs(ARGV) do
             redis.call("zrem", KEYS[1], member)

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/launchers/distributor.py

@@ -2,12 +2,12 @@ import time
 import threading
 import traceback
 
-from typing import Callable
+from typing import Callable, Type
 from inspect import isgenerator
 from urllib.parse import urlparse
 from requests import Response as Res
 
-from cobweb import setting
+from cobweb.crawlers import Crawler
 from cobweb.constant import DealModel, LogTemplate
 from cobweb.utils import LoghubDot, check_pause
 from cobweb.base import Seed, Queue, BaseItem, Request, Response, logger
@@ -23,10 +23,10 @@ class Distributor(threading.Thread):
             todo: Queue,
             done: Queue,
             upload: Queue,
-            register: Callable,
             stop: threading.Event,
             pause: threading.Event,
-            SpiderCrawler
+            callback_register: Callable,
+            SpiderCrawler: Type[Crawler]
     ):
         super().__init__()
         self.task = task
@@ -38,16 +38,16 @@ class Distributor(threading.Thread):
         self.todo = todo
         self.done = done
         self.upload = upload
-        self.register = register
+        self.callback_register = callback_register
+        self.Crawler = SpiderCrawler
 
+        from cobweb import setting
         self.time_sleep = setting.SPIDER_TIME_SLEEP
         self.thread_num = setting.SPIDER_THREAD_NUM
         self.max_retries = setting.SPIDER_MAX_RETRIES
         self.record_failed = setting.RECORD_FAILED_SPIDER
         self.loghub_dot = LoghubDot(stop=stop)  # todo: 解偶
 
-        self.Crawler = SpiderCrawler
-
         logger.debug(f"Distribute instance attrs: {self.__dict__}")
 
     def distribute(self, item, seed, _id: int):
@@ -58,6 +58,7 @@ class Distributor(threading.Thread):
             if _id == 2:
                 raise TypeError("parse function can't yield a Response instance")
             dot = isinstance(item.response, Res)
+            # TODO: 请求成功打点
             self.spider_logging(seed, item, dot=dot)
             self.process(item=item, seed=seed, callback=self.Crawler.parse, _id=2)
         elif isinstance(item, BaseItem):
@@ -77,7 +78,9 @@ class Distributor(threading.Thread):
                     try:
                         response = Response(seed, "failed", max_retries=True)
                         self.process(response, seed, self.Crawler.parse, _id=2)
-                    except:
+                    except Exception as e:
+                        msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                        logger.error(msg = msg)
                         record_failed = False
                 if not record_failed:
                     self.done.push(seed)
@@ -93,6 +96,7 @@ class Distributor(threading.Thread):
 
     @check_pause
     def spider(self):
+        # TODO: 限流措施
         if seed := self.todo.pop():
             try:
                 self.process(item=seed, seed=seed, callback=self.Crawler.request, _id=0)
@@ -102,6 +106,7 @@ class Distributor(threading.Thread):
                 if getattr(e, "response", None) and isinstance(e.response, Res):
                     url = e.response.request.url
                     status = e.response.status_code
+                    # TODO:失败请求打点
                 self.spider_logging(seed, None, error=True, url=url, status=status, msg=msg)
                 self.distribute(DealModel.fail, seed, _id=-1)
 
@@ -160,6 +165,6 @@ class Distributor(threading.Thread):
                 )
 
     def run(self):
-        self.register(self.loghub_dot.build_run, tag="LoghubDot")
+        self.callback_register(self.loghub_dot.build_run, tag="LoghubDot")
         for _ in range(self.thread_num):
-            self.register(self.spider, tag="Distributor")
+            self.callback_register(self.spider, tag="Distributor")

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/launchers/launcher.py

@@ -113,22 +113,20 @@ class Launcher:
         if not self.__WORKER_THREAD__.get(name):
             worker_thread = threading.Thread(name=name, target=func)
             self.__WORKER_THREAD__[name] = worker_thread
-            worker_thread.start()
+            # worker_thread.start()
 
     def _monitor(self):
         while not self._stop.is_set():
-            if self._pause.is_set():
-                time.sleep(15)
-                continue
-            for name, worker_thread in list(self.__WORKER_THREAD__.items()):
-                if not worker_thread.is_alive():
-                    logger.info(f"{name} thread is dead. Restarting...")
-                    func = self.__REGISTER_FUNC__[name]
-                    worker_thread = threading.Thread(name=name, target=func)
-                    self.__WORKER_THREAD__[name] = worker_thread
-                    worker_thread.start()
-                time.sleep(3)
-        logger.info("main thread close!")
+            if not self._pause.is_set():
+                for name, worker_thread in list(self.__WORKER_THREAD__.items()):
+                    if not worker_thread.is_alive():
+                        logger.debug(f"{name} thread is dead. Restarting...")
+                        func = self.__REGISTER_FUNC__[name]
+                        worker_thread = threading.Thread(name=name, target=func)
+                        self.__WORKER_THREAD__[name] = worker_thread
+                        worker_thread.start()
+            time.sleep(15)
+        logger.info("monitor thread close!")
 
     def start(self):
         self._pause.is_set()
@@ -142,7 +140,7 @@ class Launcher:
             todo=self._TODO_QUEUE_,
             done=self._DONE_QUEUE_,
             upload=self._UPLOAD_QUEUE_,
-            register=self._register
+            callback_register=self._register
         ).start()
 
         Distributor(
@@ -152,18 +150,20 @@ class Launcher:
             todo=self._TODO_QUEUE_,
             done=self._DONE_QUEUE_,
             upload=self._UPLOAD_QUEUE_,
-            register=self._register,
+            callback_register=self._register,
             stop=self._stop, pause=self._pause,
             SpiderCrawler=self.SpiderCrawler
         ).start()
 
         Uploader(
+            task=self.task, project=self.project,
             stop=self._stop, pause=self._pause,
             done=self._DONE_QUEUE_,
             upload=self._UPLOAD_QUEUE_,
-            register=self._register,
+            callback_register=self._register,
             SpiderPipeline=self.SpiderPipeline
         ).start()
 
         self._monitor()
+        logger.info("task done!")
 

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/launchers/uploader.py

@@ -1,7 +1,7 @@
 import time
 import threading
-from typing import Callable
-from cobweb import setting
+from typing import Callable, Type
+from cobweb.pipelines import Pipeline
 from cobweb.base import Queue, logger
 from cobweb.utils import check_pause
 
@@ -10,24 +10,31 @@ class Uploader(threading.Thread):
 
     def __init__(
             self,
+            task: str,
+            project: str,
             stop: threading.Event,
             pause: threading.Event,
             upload: Queue, done: Queue,
-            register: Callable,
-            SpiderPipeline
+            callback_register: Callable,
+            SpiderPipeline: Type[Pipeline]
     ):
         super().__init__()
+        self.task = task
+        self.project = project
+
         self.stop = stop
         self.pause = pause
 
         self.done = done
         self.upload = upload
-        self.register = register
+        self.callback_register = callback_register
+
+        from cobweb import setting
 
         self.upload_size = setting.UPLOAD_QUEUE_MAX_SIZE
         self.wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
 
-        self.pipeline = SpiderPipeline()
+        self.pipeline = SpiderPipeline(task=self.task, project=self.project)
 
         logger.debug(f"Uploader instance attrs: {self.__dict__}")
 
@@ -50,6 +57,7 @@ class Uploader(threading.Thread):
             for table, datas in data_info.items():
                 try:
                     self.pipeline.upload(table, datas)
+                    # TODO: 上传打点
                 except Exception as e:
                     logger.info(e)
         except Exception as e:
@@ -58,6 +66,6 @@ class Uploader(threading.Thread):
             self.done.push(seeds)
 
     def run(self):
-        self.register(self.upload_data, tag="Uploader")
+        self.callback_register(self.upload_data, tag="Uploader")
 
 

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/pipelines/__init__.py

@@ -1,3 +1,3 @@
 from .pipeline import Pipeline
-from .pipeline_console import Console
 from .pipeline_loghub import Loghub
+from .pipeline_csv import CSV

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/pipelines/pipeline.py

@@ -4,6 +4,10 @@ from cobweb.base import BaseItem
 
 class Pipeline(ABC):
 
+    def __init__(self, task, project):
+        self.task = task
+        self.project = project
+
     @abstractmethod
     def build(self, item: BaseItem) -> dict:
         pass

cobweb-launcher-3.1.12/cobweb/pipelines/pipeline_csv.py

@@ -0,0 +1,25 @@
+import os
+import csv
+
+from cobweb.base import BaseItem
+from cobweb.pipelines import Pipeline
+
+
+class CSV(Pipeline):
+
+    def __init__(self, *args, **kwargs):
+        super(CSV, self).__init__(*args, **kwargs)
+        self.log_path = rf"{os.getcwd()}\{self.project}\{self.task}\%s.csv"
+
+    def build(self, item: BaseItem):
+        return item.to_dict
+
+    def upload(self, table, datas):
+        fieldnames = datas[0].keys()
+        file_path = self.log_path % table
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        with open(file_path, mode='a', encoding='utf-8', newline="") as file:
+            writer = csv.DictWriter(file, fieldnames=fieldnames)
+            if file.tell() == 0:  # 判断文件是否为空
+                writer.writeheader()
+            writer.writerows(datas)

cobweb-launcher-3.1.12/cobweb/pipelines/pipeline_loghub.py

@@ -0,0 +1,54 @@
+import os
+import json
+
+from cobweb.base import BaseItem
+from cobweb.pipelines import Pipeline
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+from collections import defaultdict
+
+
+class Loghub(Pipeline):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = LogClient(
+            endpoint=os.getenv("LOGHUB_ENDPOINT"),
+            accessKeyId=os.getenv("LOGHUB_ACCESS_KEY"),
+            accessKey=os.getenv("LOGHUB_SECRET_KEY")
+        )
+        self.project = os.getenv("LOGHUB_PROJECT")
+        self.source = os.getenv("LOGHUB_SOURCE")
+        self.topic = os.getenv("LOGHUB_TOPIC")
+
+    def build(self, item: BaseItem):
+        log_item = LogItem()
+        temp = item.to_dict
+        for key, value in temp.items():
+            if not isinstance(value, str):
+                temp[key] = json.dumps(value, ensure_ascii=False)
+        contents = sorted(temp.items())
+        log_item.set_contents(contents)
+        return (
+            log_item,
+            item.baseitem_topic or self.topic,
+            item.baseitem_source or self.source,
+            item.baseitem_project or self.project,
+        )
+
+    def upload(self, table, datas):
+
+        upload_items = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
+
+        for log_item, topic, source, project in datas:
+            upload_items[project][source][topic].append(log_item)
+
+        for request in [
+            PutLogsRequest(
+                logstore=table, project=project,
+                topic=topic, source=source,
+                logitems=log_items, compress=True
+            ) for project, sources in upload_items.items()
+            for source, topics in sources.items()
+            for topic, log_items in topics.items()
+        ]:
+            self.client.put_logs(request=request)

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/schedulers/scheduler.py

@@ -1,6 +1,6 @@
 import threading
 
-from cobweb import setting
+
 from typing import Callable
 from cobweb.base import Queue
 from abc import ABC, abstractmethod
@@ -20,11 +20,12 @@ class Scheduler(ABC, threading.Thread):
             todo: Queue,
             done: Queue,
             upload: Queue,
-            register: Callable
+            callback_register: Callable
     ):
         super().__init__()
         self.task = task
         self.project = project
+        from cobweb import setting
 
         self.task_model = setting.TASK_MODEL
         self.seed_reset_seconds = setting.SEED_RESET_SECONDS
@@ -47,7 +48,7 @@ class Scheduler(ABC, threading.Thread):
         self.done = done
         self.upload = upload
 
-        self.register = register
+        self.callback_register = callback_register
 
         self.working_seeds = dict()
 

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/schedulers/scheduler_with_redis.py

@@ -22,16 +22,16 @@ class RedisScheduler(Scheduler):
             todo: Queue,
             done: Queue,
             upload: Queue,
-            register: Callable
+            callback_register: Callable
     ):
-        super().__init__(task, project, stop, pause, new, todo, done, upload, register)
+        super().__init__(task, project, stop, pause, new, todo, done, upload, callback_register)
         self.todo_key = "{%s:%s}:todo" % (project, task)
         self.done_key = "{%s:%s}:done" % (project, task)
         self.fail_key = "{%s:%s}:fail" % (project, task)
         self.heartbeat_key = "heartbeat:%s_%s" % (project, task)
         self.speed_control_key = "speed_control:%s_%s" % (project, task)
         self.reset_lock_key = "lock:reset:%s_%s" % (project, task)
-        self.redis_queue_empty_event = threading.Event()
+        # self.redis_queue_empty_event = threading.Event()
         self.db = ApiDB() if use_api else RedisDB()
 
     @check_pause
@@ -126,11 +126,11 @@ class RedisScheduler(Scheduler):
     def run(self):
         start_time = int(time.time())
 
-        self.register(self.reset, tag="scheduler")
-        self.register(self.insert, tag="scheduler")
-        self.register(self.delete, tag="scheduler")
-        self.register(self.refresh, tag="scheduler")
-        self.register(self.schedule, tag="scheduler")
+        self.callback_register(self.reset, tag="scheduler")
+        self.callback_register(self.insert, tag="scheduler")
+        self.callback_register(self.delete, tag="scheduler")
+        self.callback_register(self.refresh, tag="scheduler")
+        self.callback_register(self.schedule, tag="scheduler")
 
         while not self.stop.is_set():
             working_count = len(self.working_seeds.keys())
@@ -149,13 +149,12 @@ class RedisScheduler(Scheduler):
                         self.pause.clear()
                     else:
                         logger.info("pause! waiting for resume...")
+                elif all_count:
+                    logger.info(f"todo seeds count: {todo_count}, queue length: {all_count}")
+                    self.pause.clear()
                 else:
-                    if all_count:
-                        logger.info(f"todo seeds count: {todo_count}, queue length: {all_count}")
-                        self.pause.clear()
-                    else:
-                        logger.info("Done! pause set...")
-                        self.pause.set()
+                    logger.info("TODO queue is empty! pause set...")
+                    self.pause.set()
             else:
                 if self.pause.is_set():
                     self.pause.clear()
@@ -171,6 +170,3 @@ class RedisScheduler(Scheduler):
                 ))
 
             time.sleep(30)
-
-        logger.info("Scheduler Done!")
-

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/setting.py

@@ -1,37 +1,8 @@
-import os
-
-# redis db config
-REDIS_CONFIG = {
-    "host": os.getenv("REDIS_HOST"),
-    "password": os.getenv("REDIS_PASSWORD"),
-    "port": int(os.getenv("REDIS_PORT", 6379)),
-    "db": int(os.getenv("REDIS_DB", 0)),
-}
-
-# loghub db config
-LOGHUB_TOPIC = os.getenv("LOGHUB_TOPIC")
-LOGHUB_SOURCE = os.getenv("LOGHUB_SOURCE")
-LOGHUB_PROJECT = os.getenv("LOGHUB_PROJECT")
-LOGHUB_CONFIG = {
-    "endpoint": os.getenv("LOGHUB_ENDPOINT"),
-    "accessKeyId": os.getenv("LOGHUB_ACCESS_KEY"),
-    "accessKey": os.getenv("LOGHUB_SECRET_KEY")
-}
-
-# # oss util config
-OSS_BUCKET = os.getenv("OSS_BUCKET")
-OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
-OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
-OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
-OSS_CHUNK_SIZE = 10 * 1024 ** 2
-OSS_MIN_UPLOAD_SIZE = 1024
-
-
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
 
 # 数据管道
-PIPELINE = "cobweb.pipelines.Console"
+PIPELINE = "cobweb.pipelines.CSV"
 
 # 调度器
 SCHEDULER = "cobweb.schedulers.RedisScheduler"
@@ -61,20 +32,28 @@ DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
 SPIDER_TIME_SLEEP = 10
-RECORD_FAILED_SPIDER = False
+RECORD_FAILED_SPIDER = True
 
 SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
 TIME_WINDOW = 60   # 频控固定时间窗口（秒）
 
-# 任务模式
-TASK_MODEL = 0  # 0:单次，1:常驻
+# 任务模式, 0:单次，1:常驻
+TASK_MODEL = 0
+
+# 流控措施, 0:关闭，1:开启
+SPEED_CONTROL = 1
+
 
-# 流控措施
-SPEED_CONTROL = 1  # 0:关闭，1:开启
+# scheduler redis config
+# os.getenv("REDIS_HOST", "127.0.0.1")
+# os.getenv("REDIS_PASSWORD")
+# os.getenv("REDIS_PORT", 6379)
+# os.getenv("REDIS_DB", 0)
 
-# # bloom过滤器
-# CAPACITY = 100000000
-# ERROR_RATE = 0.001
-# FILTER_FIELD = "url"
-# 文件下载响应类型过滤
-# FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]
+# loghub pipeline config
+# os.getenv("LOGHUB_ENDPOINT"),
+# os.getenv("LOGHUB_ACCESS_KEY"),
+# os.getenv("LOGHUB_SECRET_KEY")
+# os.getenv("LOGHUB_PROJECT")
+# os.getenv("LOGHUB_SOURCE")
+# os.getenv("LOGHUB_TOPIC")

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/utils/__init__.py

@@ -1,6 +1,6 @@
 # from .oss import OssUtil
-from .tools import *
-from .bloom import BloomFilter
+# from .bloom import BloomFilter
 from .dotting import LoghubDot
 from .decorators import check_pause
+from .tools import md5, dynamic_load_class
 

cobweb-launcher-3.1.12/cobweb/utils/bloom.py

@@ -0,0 +1,58 @@
+# import math
+# import time
+#
+# import mmh3
+# import redis
+# from cobweb import setting
+#
+#
+# class BloomFilter:
+#
+#     def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
+#         redis_config = redis_config or setting.REDIS_CONFIG
+#         capacity = capacity or setting.CAPACITY
+#         error_rate = error_rate or setting.ERROR_RATE
+#         redis_config['db'] = 3
+#
+#         self.key = key
+#
+#         pool = redis.ConnectionPool(**redis_config)
+#         self._client = redis.Redis(connection_pool=pool)
+#         self.bit_size = self.get_bit_size(capacity, error_rate)
+#         self.hash_count = self.get_hash_count(self.bit_size, capacity)
+#         self._init_bloom_key()
+#
+#     def add(self, value):
+#         for seed in range(self.hash_count):
+#             result = mmh3.hash(value, seed) % self.bit_size
+#             self._client.setbit(self.key, result, 1)
+#         return True
+#
+#     def exists(self, value):
+#         if not self._client.exists(self.key):
+#             return False
+#         for seed in range(self.hash_count):
+#             result = mmh3.hash(value, seed) % self.bit_size
+#             if not self._client.getbit(self.key, result):
+#                 return False
+#         return True
+#
+#     def _init_bloom_key(self):
+#         lua_script = """
+#         redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
+#         redis.call("EXPIRE", KEYS[1], 604800)
+#         """
+#         if self._client.exists(self.key):
+#             return True
+#         execute = self._client.register_script(lua_script)
+#         execute(keys=[self.key], args=[self.bit_size-1, 1])
+#
+#     @classmethod
+#     def get_bit_size(cls, n, p):
+#         return int(-(n * math.log(p)) / (math.log(2) ** 2))
+#
+#     @classmethod
+#     def get_hash_count(cls, m, n):
+#         return int((m / n) * math.log(2))
+#
+#

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/utils/dotting.py

@@ -1,21 +1,23 @@
+import os
 import json
 import time
 
 from aliyun.log import LogClient, LogItem, PutLogsRequest
-
 from cobweb.base import Queue, logger
-from cobweb import setting
 
 
 class LoghubDot:
 
     def __init__(self, stop):
         self.stop = stop
-        self.client = LogClient(**setting.LOGHUB_CONFIG)
         self.queue = Queue()
+        self.client = LogClient(
+            endpoint=os.getenv("LOGHUB_ENDPOINT"),
+            accessKeyId=os.getenv("LOGHUB_ACCESS_KEY"),
+            accessKey=os.getenv("LOGHUB_SECRET_KEY")
+        )
 
     def build(self, topic, **kwargs):
-
         temp = {}
         log_item = LogItem()
         for key, value in kwargs.items():

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb/utils/tools.py

@@ -1,5 +1,6 @@
 import re
 import hashlib
+import inspect
 from typing import Union
 from importlib import import_module
 
@@ -10,18 +11,6 @@ def md5(text: Union[str, bytes]) -> str:
     return hashlib.md5(text).hexdigest()
 
 
-def build_path(site, url, file_type):
-    return f"{site}/{md5(url)}.{file_type}"
-
-
-def format_size(content_length: int) -> str:
-    units = ["KB", "MB", "GB", "TB"]
-    for i in range(4):
-        num = content_length / (1024 ** (i + 1))
-        if num < 1024:
-            return f"{round(num, 2)} {units[i]}"
-
-
 def dynamic_load_class(model_info):
     if isinstance(model_info, str):
         if "import" in model_info:
@@ -35,8 +24,7 @@ def dynamic_load_class(model_info):
             model = import_module(model_path)
             class_object = getattr(model, class_name)
         return class_object
+    elif inspect.isclass(model_info):
+        return model_info
     raise TypeError()
 
-
-# def download_log_info(item:dict) -> str:
-#     return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.11
+Version: 3.1.12
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb_launcher.egg-info/SOURCES.txt

@@ -7,7 +7,7 @@ cobweb/setting.py
 cobweb/base/__init__.py
 cobweb/base/common_queue.py
 cobweb/base/item.py
-cobweb/base/log.py
+cobweb/base/logger.py
 cobweb/base/request.py
 cobweb/base/response.py
 cobweb/base/seed.py
@@ -24,7 +24,7 @@ cobweb/launchers/launcher.py
 cobweb/launchers/uploader.py
 cobweb/pipelines/__init__.py
 cobweb/pipelines/pipeline.py
-cobweb/pipelines/pipeline_console.py
+cobweb/pipelines/pipeline_csv.py
 cobweb/pipelines/pipeline_loghub.py
 cobweb/schedulers/__init__.py
 cobweb/schedulers/scheduler.py

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/cobweb_launcher.egg-info/requires.txt

@@ -1,5 +1,3 @@
 requests>=2.19.1
-oss2>=2.18.1
 redis>=4.4.4
 aliyun-log-python-sdk
-mmh3

{cobweb-launcher-3.1.11 → cobweb-launcher-3.1.12}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="3.1.11",
+    version="3.1.12",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",
@@ -14,7 +14,7 @@ setup(
     description="spider_hole",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    install_requires=["requests>=2.19.1", "oss2>=2.18.1", "redis>=4.4.4", "aliyun-log-python-sdk", "mmh3"],
+    install_requires=["requests>=2.19.1", "redis>=4.4.4", "aliyun-log-python-sdk"],
     classifiers=[
         "Programming Language :: Python :: 3",
     ],

cobweb-launcher-3.1.11/cobweb/crawlers/crawler.py

@@ -1,28 +0,0 @@
-import json
-from typing import Union
-from cobweb.base import (
-    Seed,
-    BaseItem,
-    Request,
-    Response,
-    ConsoleItem,
-)
-
-
-class Crawler:
-
-    @staticmethod
-    def request(seed: Seed) -> Union[Request, BaseItem]:
-        yield Request(seed.url, seed, timeout=5)
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        response = item.download()
-        yield Response(item.seed, response, **item.to_dict)
-
-    @staticmethod
-    def parse(item: Response) -> BaseItem:
-        upload_item = item.to_dict
-        upload_item["text"] = item.response.text
-        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
-

cobweb-launcher-3.1.11/cobweb/pipelines/pipeline_console.py

@@ -1,22 +0,0 @@
-from cobweb.base import ConsoleItem, logger
-from cobweb.constant import LogTemplate
-from cobweb.pipelines import Pipeline
-
-
-class Console(Pipeline):
-
-    def build(self, item: ConsoleItem):
-        return {
-            "seed": item.seed.to_dict,
-            "data": item.to_dict
-        }
-
-    def upload(self, table, datas):
-        for data in datas:
-            parse_detail = LogTemplate.log_info(data["data"])
-            if len(parse_detail) > 500:
-                parse_detail = parse_detail[:500] + " ...\n" + " " * 12 + "-- Text is too long and details are omitted!"
-            logger.info(LogTemplate.console_item.format(
-                seed_detail=LogTemplate.log_info(data["seed"]),
-                parse_detail=parse_detail
-            ))

cobweb-launcher-3.1.11/cobweb/pipelines/pipeline_loghub.py

@@ -1,34 +0,0 @@
-import json
-
-from cobweb import setting
-from cobweb.base import BaseItem
-from cobweb.pipelines import Pipeline
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-
-
-class Loghub(Pipeline):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.client = LogClient(**setting.LOGHUB_CONFIG)
-
-    def build(self, item: BaseItem):
-        log_item = LogItem()
-        temp = item.to_dict
-        for key, value in temp.items():
-            if not isinstance(value, str):
-                temp[key] = json.dumps(value, ensure_ascii=False)
-        contents = sorted(temp.items())
-        log_item.set_contents(contents)
-        return log_item
-
-    def upload(self, table, datas):
-        request = PutLogsRequest(
-            project=setting.LOGHUB_PROJECT,
-            logstore=table,
-            topic=setting.LOGHUB_TOPIC,
-            source=setting.LOGHUB_SOURCE,
-            logitems=datas,
-            compress=True
-        )
-        self.client.put_logs(request=request)

cobweb-launcher-3.1.11/cobweb/utils/bloom.py

@@ -1,58 +0,0 @@
-import math
-import time
-
-import mmh3
-import redis
-from cobweb import setting
-
-
-class BloomFilter:
-
-    def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
-        redis_config = redis_config or setting.REDIS_CONFIG
-        capacity = capacity or setting.CAPACITY
-        error_rate = error_rate or setting.ERROR_RATE
-        redis_config['db'] = 3
-
-        self.key = key
-
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
-        self.bit_size = self.get_bit_size(capacity, error_rate)
-        self.hash_count = self.get_hash_count(self.bit_size, capacity)
-        self._init_bloom_key()
-
-    def add(self, value):
-        for seed in range(self.hash_count):
-            result = mmh3.hash(value, seed) % self.bit_size
-            self._client.setbit(self.key, result, 1)
-        return True
-
-    def exists(self, value):
-        if not self._client.exists(self.key):
-            return False
-        for seed in range(self.hash_count):
-            result = mmh3.hash(value, seed) % self.bit_size
-            if not self._client.getbit(self.key, result):
-                return False
-        return True
-
-    def _init_bloom_key(self):
-        lua_script = """
-        redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
-        redis.call("EXPIRE", KEYS[1], 604800)
-        """
-        if self._client.exists(self.key):
-            return True
-        execute = self._client.register_script(lua_script)
-        execute(keys=[self.key], args=[self.bit_size-1, 1])
-
-    @classmethod
-    def get_bit_size(cls, n, p):
-        return int(-(n * math.log(p)) / (math.log(2) ** 2))
-
-    @classmethod
-    def get_hash_count(cls, m, n):
-        return int((m / n) * math.log(2))
-
-