cobweb-launcher 3.1.4__tar.gz → 3.1.6__tar.gz

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.4
+Version: 3.1.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/cobweb/__init__.py

@@ -1,2 +1,4 @@
 from .launchers import Launcher
 from .constant import CrawlerModel
+from .pipelines import Pipeline
+from .crawlers import Crawler

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/cobweb/db/api_db.py

@@ -71,7 +71,7 @@ class ApiDB:
     def auto_incr(self, name, t=15, limit=1000) -> bool:
         return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
 
-    def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
+    def members(self, name, score, start=0, count=1000, _min="-inf", _max="+inf"):
         return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
 
     def done(self, name: list, *values):

cobweb-launcher-3.1.6/cobweb/launchers/__init__.py

@@ -0,0 +1,4 @@
+from .launcher import Launcher
+from .uploader import Uploader
+from .distributor import Distributor
+

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/cobweb/launchers/distributor.py

@@ -1,22 +1,16 @@
 import time
 import threading
 import traceback
-from inspect import isgenerator
+
 from typing import Callable
+from inspect import isgenerator
 from urllib.parse import urlparse
 from requests import Response as Res
 
 from cobweb import setting
 from cobweb.constant import DealModel, LogTemplate
-from cobweb.base import (
-    Seed,
-    Queue,
-    BaseItem,
-    Request,
-    Response,
-    logger
-)
 from cobweb.utils import LoghubDot, check_pause
+from cobweb.base import Seed, Queue, BaseItem, Request, Response, logger
 
 
 class Distributor(threading.Thread):

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/cobweb/launchers/uploader.py

@@ -27,7 +27,7 @@ class Uploader(threading.Thread):
         self.upload_size = setting.UPLOAD_QUEUE_MAX_SIZE
         self.wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
 
-        self.Pipeline = SpiderPipeline
+        self.pipeline = SpiderPipeline()
 
         logger.debug(f"Uploader instance attrs: {self.__dict__}")
 
@@ -45,11 +45,11 @@ class Uploader(threading.Thread):
                 if not item:
                     break
                 seeds.append(item.seed)
-                data = self.Pipeline.build(item)
+                data = self.pipeline.build(item)
                 data_info.setdefault(item.table, []).append(data)
             for table, datas in data_info.items():
                 try:
-                    self.Pipeline.upload(table, datas)
+                    self.pipeline.upload(table, datas)
                 except Exception as e:
                     logger.info(e)
         except Exception as e:

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/cobweb/schedulers/scheduler_with_redis.py

@@ -7,7 +7,7 @@ from cobweb.utils import check_pause
 from cobweb.base import Queue, Seed, logger
 from cobweb.constant import LogTemplate
 from .scheduler import Scheduler
-use_api = bool(int(os.getenv("REDIS_API", 0)))
+use_api = bool(int(os.getenv("REDIS_API_HOST", 0)))
 
 
 class RedisScheduler(Scheduler):
@@ -153,10 +153,10 @@ class RedisScheduler(Scheduler):
                 else:
                     if all_count:
                         logger.info(f"todo seeds count: {todo_count}, queue length: {all_count}")
-                        self.pause.is_set()
+                        self.pause.clear()
                     else:
                         logger.info("Done! pause set...")
-                        self.pause.clear()
+                        self.pause.set()
             else:
                 if self.pause.is_set():
                     self.pause.clear()

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/cobweb/setting.py

@@ -18,7 +18,7 @@ LOGHUB_CONFIG = {
     "accessKey": os.getenv("LOGHUB_SECRET_KEY")
 }
 
-# oss util config
+# # oss util config
 OSS_BUCKET = os.getenv("OSS_BUCKET")
 OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
 OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
@@ -72,9 +72,9 @@ TASK_MODEL = 0  # 0:单次，1:常驻
 # 流控措施
 SPEED_CONTROL = 1  # 0:关闭，1:开启
 
-# bloom过滤器
-CAPACITY = 100000000
-ERROR_RATE = 0.001
-FILTER_FIELD = "url"
+# # bloom过滤器
+# CAPACITY = 100000000
+# ERROR_RATE = 0.001
+# FILTER_FIELD = "url"
 # 文件下载响应类型过滤
 # FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.4
+Version: 3.1.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/cobweb_launcher.egg-info/SOURCES.txt

@@ -27,8 +27,6 @@ cobweb/pipelines/pipeline.py
 cobweb/pipelines/pipeline_console.py
 cobweb/pipelines/pipeline_loghub.py
 cobweb/schedulers/__init__.py
-cobweb/schedulers/launcher_air.py
-cobweb/schedulers/launcher_api.py
 cobweb/schedulers/scheduler.py
 cobweb/schedulers/scheduler_with_redis.py
 cobweb/utils/__init__.py

{cobweb-launcher-3.1.4 → cobweb-launcher-3.1.6}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="3.1.4",
+    version="3.1.6",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",

cobweb-launcher-3.1.4/cobweb/launchers/__init__.py

@@ -1,9 +0,0 @@
-# from .launcher_air import LauncherAir
-# from .launcher_pro import LauncherPro
-# from .launcher_api import LauncherApi
-
-
-from .launcher import Launcher
-from .uploader import Uploader
-from .distributor import Distributor
-

cobweb-launcher-3.1.4/cobweb/schedulers/launcher_air.py

@@ -1,93 +0,0 @@
-import time
-
-from cobweb.base import logger
-from cobweb.constant import LogTemplate
-from .launcher import Launcher, check_pause
-
-
-class LauncherAir(Launcher):
-
-    # def _scheduler(self):
-    #     if self.start_seeds:
-    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-
-    @check_pause
-    def _insert(self):
-        new_seeds = {}
-        del_seeds = set()
-        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-        for _ in range(self._new_queue_max_size):
-            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed_tuple:
-                break
-            seed, new_seed = seed_tuple
-            new_seeds[new_seed.to_string] = new_seed.params.priority
-            del_seeds.add(seed.to_string)
-        if new_seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(new_seeds)
-        if del_seeds:
-            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
-        if status:
-            time.sleep(self._new_queue_wait_seconds)
-
-    @check_pause
-    def _delete(self):
-        seeds = []
-        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-        for _ in range(self._done_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['done'].pop()
-            if not seed:
-                break
-            seeds.append(seed.to_string)
-
-        if seeds:
-            self._remove_doing_seeds(seeds)
-
-        if status:
-            time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-
-        check_emtpy_times = 0
-
-        while not self._stop.is_set():
-
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-
-            if queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
-                if self._pause.is_set():
-                    check_emtpy_times = 0
-                    if not self._task_model:
-                        logger.info("Done! Ready to close thread...")
-                        self._stop.set()
-                elif check_emtpy_times > 2:
-                    self.__DOING__ = {}
-                    self._pause.set()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            elif self._pause.is_set():
-                self._pause.clear()
-                self._execute()
-            else:
-                logger.info(LogTemplate.launcher_air_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-                ))
-
-            time.sleep(pooling_wait_seconds)
-
-

cobweb-launcher-3.1.4/cobweb/schedulers/launcher_api.py

@@ -1,225 +0,0 @@
-import time
-import threading
-
-from cobweb.db import ApiDB
-from cobweb.base import Seed, logger
-from cobweb.constant import DealModel, LogTemplate
-from .launcher import Launcher, check_pause
-
-
-class LauncherApi(Launcher):
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__(task, project, custom_setting, **kwargs)
-        self._db = ApiDB()
-
-        self._todo_key = "{%s:%s}:todo" % (project, task)
-        self._done_key = "{%s:%s}:done" % (project, task)
-        self._fail_key = "{%s:%s}:fail" % (project, task)
-        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
-
-        self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
-        self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
-        self._speed_control_key = "speed_control:%s_%s" % (project, task)
-
-        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
-
-        # self._bf_key = "bloom_%s_%s" % (project, task)
-        # self._bf = BloomFilter(self._bf_key)
-
-        self._heartbeat_start_event = threading.Event()
-        self._redis_queue_empty_event = threading.Event()
-
-    @property
-    def heartbeat(self):
-        return self._db.exists(self._heartbeat_key)
-
-    def statistics(self, key, count):
-        if not self._task_model and not self._db.exists(key):
-            self._db.setex(key, 86400 * 30, int(count))
-        else:
-            self._db.incrby(key, count)
-
-    def _get_seed(self) -> Seed:
-        """
-        从队列中获取种子（频控）
-        设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
-        :return: True -> 种子, False -> None
-        """
-        if (self._speed_control and self.__LAUNCHER_QUEUE__["todo"].length and
-                not self._db.auto_incr(self._speed_control_key, t=self._time_window, limit=self._spider_max_count)):
-            expire_time = self._db.ttl(self._speed_control_key)
-            if isinstance(expire_time, int) and expire_time <= -1:
-                self._db.delete(self._speed_control_key)
-            elif isinstance(expire_time, int):
-                logger.info(f"Too fast! Please wait {expire_time} seconds...")
-                time.sleep(expire_time / 2)
-            return None
-        seed = self.__LAUNCHER_QUEUE__["todo"].pop()
-        return seed
-
-    @check_pause
-    def _execute_heartbeat(self):
-        if self._heartbeat_start_event.is_set():
-            self._db.setex(self._heartbeat_key, 5)
-        time.sleep(3)
-
-    @check_pause
-    def _reset(self):
-        """
-        检查过期种子，重新添加到redis缓存中
-        """
-        reset_wait_seconds = 30
-        if self._db.lock(self._reset_lock_key, t=120):
-
-            _min = -int(time.time()) + self._seed_reset_seconds \
-                if self.heartbeat else "-inf"
-
-            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
-
-            if not self.heartbeat:
-                self._heartbeat_start_event.set()
-
-            self._db.delete(self._reset_lock_key)
-
-        time.sleep(reset_wait_seconds)
-
-    @check_pause
-    def _scheduler(self):
-        """
-        调度任务，获取redis队列种子，同时添加到doing字典中
-        """
-        if not self._db.zcount(self._todo_key, 0, "(1000"):
-            time.sleep(self._scheduler_wait_seconds)
-        elif self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
-            time.sleep(self._todo_queue_full_wait_seconds)
-        else:
-            members = self._db.members(
-                self._todo_key, int(time.time()),
-                count=self._todo_queue_size,
-                _min=0, _max="(1000"
-            )
-            for member, priority in members:
-                seed = Seed(member, priority=priority)
-                self.__LAUNCHER_QUEUE__['todo'].push(seed)
-                self.__DOING__[seed.to_string] = seed.params.priority
-
-    @check_pause
-    def _insert(self):
-        """
-        添加新种子到redis队列中
-        """
-        new_seeds = {}
-        del_seeds = set()
-        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-        for _ in range(self._new_queue_max_size):
-            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed_tuple:
-                break
-            seed, new_seed = seed_tuple
-            new_seeds[new_seed.to_string] = new_seed.params.priority
-            del_seeds.add(seed)
-        if new_seeds:
-            self._db.zadd(self._todo_key, new_seeds, nx=True)
-        if del_seeds:
-            self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
-        if status:
-            time.sleep(self._new_queue_wait_seconds)
-
-    @check_pause
-    def _refresh(self):
-        """
-        刷新doing种子过期时间，防止reset重新消费
-        """
-        if self.__DOING__:
-            refresh_time = int(time.time())
-            seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
-            self._db.zadd(self._todo_key, item=seeds, xx=True)
-        time.sleep(15)
-
-    @check_pause
-    def _delete(self):
-        """
-        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
-        """
-        # seed_info = {"count": 0, "failed": [], "succeed": [], "common": []}
-
-        seed_list = []
-        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-        for _ in range(self._done_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['done'].pop()
-            if not seed:
-                break
-            seed_list.append(seed.to_string)
-
-        if seed_list:
-
-            self._db.zrem(self._todo_key, *seed_list)
-            self._remove_doing_seeds(seed_list)
-
-        if status:
-            time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-        wait_scheduler_execute = True
-        check_emtpy_times = 0
-        while not self._stop.is_set():
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-                    wait_scheduler_execute = False
-
-            if queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
-                if self._pause.is_set():
-                    check_emtpy_times = 0
-                    if not self._task_model and (
-                            not wait_scheduler_execute or
-                            int(time.time()) - self._app_time > self._before_scheduler_wait_seconds
-                    ):
-                        logger.info("Done! ready to close thread...")
-                        self._stop.set()
-
-                    elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
-                        logger.info(f"Recovery {self.task} task run！")
-                        self._pause.clear()
-                        self._execute()
-                    else:
-                        logger.info("pause! waiting for resume...")
-                elif check_emtpy_times > 2:
-                    self.__DOING__ = {}
-                    seed_count = self._db.zcard(self._todo_key)
-                    logger.info(f"队列剩余种子数:{seed_count}")
-                    if not seed_count:
-                        logger.info("Done! pause set...")
-                        self._pause.set()
-                    else:
-                        self._pause.clear()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            else:
-                if self._pause.is_set():
-                    self._pause.clear()
-                logger.info(LogTemplate.launcher_pro_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    redis_seed_count=self._db.zcount(self._todo_key, "-inf", "+inf"),
-                    redis_todo_len=self._db.zcount(self._todo_key, 0, "(1000"),
-                    redis_doing_len=self._db.zcount(self._todo_key, "-inf", "(0"),
-                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-                ))
-
-            time.sleep(pooling_wait_seconds)
-
-        logger.info("Done! Ready to close thread...")
-