cobweb-launcher 1.2.50__tar.gz → 1.2.51__tar.gz

{cobweb-launcher-1.2.50/cobweb_launcher.egg-info → cobweb-launcher-1.2.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.50
+Version: 1.2.51
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.50 → cobweb-launcher-1.2.51}/cobweb/launchers/launcher.py

@@ -100,6 +100,7 @@ class Launcher(threading.Thread):
         self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
         self._spider_max_count = setting.SPIDER_MAX_COUNT
         self._time_window = setting.TIME_WINDOW
+        self._speed_control = setting.SPEED_CONTROL
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL

{cobweb-launcher-1.2.50 → cobweb-launcher-1.2.51}/cobweb/launchers/launcher_api.py

@@ -46,7 +46,7 @@ class LauncherApi(Launcher):
         设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
         :return: True -> 种子, False -> None
         """
-        if (self.__LAUNCHER_QUEUE__["todo"].length and
+        if (self._speed_control and self.__LAUNCHER_QUEUE__["todo"].length and
                 not self._db.auto_incr(self._speed_control_key, t=self._time_window, limit=self._spider_max_count)):
             expire_time = self._db.ttl(self._speed_control_key)
             logger.info(f"Too fast! Please wait {expire_time} seconds...")

{cobweb-launcher-1.2.50 → cobweb-launcher-1.2.51}/cobweb/launchers/launcher_pro.py

@@ -3,7 +3,7 @@ import threading
 
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
-from cobweb.utils import BloomFilter
+# from cobweb.utils import BloomFilter
 from cobweb.constant import DealModel, LogTemplate
 from .launcher import Launcher, check_pause
 
@@ -43,17 +43,18 @@ class LauncherPro(Launcher):
             self._db._client.incrby(key, count)
 
     def _get_seed(self) -> Seed:
-        spider_speed = self._db._client.get(self._speed_control_key)
-        if int(spider_speed or 0) > self._spider_max_count:
-            expire_time = self._db.ttl(self._speed_control_key)
-            if expire_time == -1:
-                self._db.delete(self._speed_control_key)
-            else:
-                logger.info(f"Too fast! Please wait {expire_time} seconds...")
-                time.sleep(expire_time / 2)
-            return None
+        if self._speed_control:
+            spider_speed = self._db._client.get(self._speed_control_key)
+            if int(spider_speed or 0) > self._spider_max_count:
+                expire_time = self._db.ttl(self._speed_control_key)
+                if expire_time == -1:
+                    self._db.delete(self._speed_control_key)
+                else:
+                    logger.info(f"Too fast! Please wait {expire_time} seconds...")
+                    time.sleep(expire_time / 2)
+                return None
         seed = self.__LAUNCHER_QUEUE__["todo"].pop()
-        if seed and not self._db.lock(self._speed_control_key, t=self._time_window):
+        if self._speed_control and seed and not self._db.lock(self._speed_control_key, t=self._time_window):
             self._db._client.incrby(self._speed_control_key, 1)
         return seed
 

{cobweb-launcher-1.2.50 → cobweb-launcher-1.2.51}/cobweb/setting.py

@@ -65,6 +65,8 @@ TIME_WINDOW = 60   # 频控固定时间窗口（秒）
 # 任务模式
 TASK_MODEL = 0  # 0:单次，1:常驻
 
+# 流控措施
+SPEED_CONTROL = 1  # 0:关闭，1:开启
 
 # bloom过滤器
 CAPACITY = 100000000

{cobweb-launcher-1.2.50 → cobweb-launcher-1.2.51/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.50
+Version: 1.2.51
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.50 → cobweb-launcher-1.2.51}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.50",
+    version="1.2.51",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",