cobweb-launcher 1.2.50__py3-none-any.whl → 1.2.52__py3-none-any.whl

cobweb/db/redis_db.py

@@ -6,60 +6,82 @@ class RedisDB:
 
     def __init__(self, **kwargs):
         redis_config = kwargs or setting.REDIS_CONFIG
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
+        # pool = redis.ConnectionPool(**redis_config)
+        self.pool = redis.ConnectionPool(
+            max_connections=25,
+            **redis_config
+        )
+        
+    def get_connection(self):
+        return redis.StrictRedis(connection_pool=self.pool)
+        # self._client = redis.Redis(connection_pool=pool)
 
     def setnx(self, name, value=""):
-        return self._client.setnx(name, value)
+        with self.get_connection() as client:
+            client.setnx(name, value)
 
     def setex(self, name, t, value=""):
-        return self._client.setex(name, t, value)
+        with self.get_connection() as client:
+            client.setex(name, t, value)
 
     def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        return self._client.expire(name, t, nx, xx, gt, lt)
+        with self.get_connection() as client:
+            client.expire(name, t, nx, xx, gt, lt)
 
     def ttl(self, name):
-        return self._client.ttl(name)
+        with self.get_connection() as client:
+            return client.ttl(name)
 
     def delete(self, name):
-        return self._client.delete(name)
+        with self.get_connection() as client:
+            return client.delete(name)
 
     def exists(self, *name) -> bool:
-        return self._client.exists(*name)
+        with self.get_connection() as client:
+            return client.exists(*name)
 
     def sadd(self, name, value):
-        return self._client.sadd(name, value)
+        with self.get_connection() as client:
+            return client.sadd(name, value)
 
     def zcard(self, name) -> bool:
-        return self._client.zcard(name)
+        with self.get_connection() as client:
+            return client.zcard(name)
 
     def zadd(self, name, item: dict, **kwargs):
-        return self._client.zadd(name, item, **kwargs)
+        with self.get_connection() as client:
+            return client.zadd(name, item, **kwargs)
 
     def zrem(self, name, *value):
-        return self._client.zrem(name, *value)
+        with self.get_connection() as client:
+            return client.zrem(name, *value)
 
     def zcount(self, name, _min, _max):
-        return self._client.zcount(name, _min, _max)
+        with self.get_connection() as client:
+            return client.zcount(name, _min, _max)
 
     # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
-    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+    #     with self.get_connection() as client:
+    #        return client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
 
     def lua(self, script: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        return self._client.eval(script, keys_count, *keys, *args)
+        with self.get_connection() as client:
+            return client.eval(script, keys_count, *keys, *args)
 
     def lua_sha(self, sha1: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        return self._client.evalsha(sha1, keys_count, *keys, *args)
+        with self.get_connection() as client:
+            return client.evalsha(sha1, keys_count, *keys, *args)
 
     def execute_lua(self, lua_script: str, keys: list, *args):
-        execute = self._client.register_script(lua_script)
-        return execute(keys=keys, args=args)
+        with self.get_connection() as client:
+            execute = client.register_script(lua_script)
+            return execute(keys=keys, args=args)
 
     def lock(self, key, t=15) -> bool:
         lua_script = """

cobweb/launchers/launcher.py

@@ -100,6 +100,7 @@ class Launcher(threading.Thread):
         self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
         self._spider_max_count = setting.SPIDER_MAX_COUNT
         self._time_window = setting.TIME_WINDOW
+        self._speed_control = setting.SPEED_CONTROL
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL

cobweb/launchers/launcher_api.py

@@ -46,7 +46,7 @@ class LauncherApi(Launcher):
         设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
         :return: True -> 种子, False -> None
         """
-        if (self.__LAUNCHER_QUEUE__["todo"].length and
+        if (self._speed_control and self.__LAUNCHER_QUEUE__["todo"].length and
                 not self._db.auto_incr(self._speed_control_key, t=self._time_window, limit=self._spider_max_count)):
             expire_time = self._db.ttl(self._speed_control_key)
             logger.info(f"Too fast! Please wait {expire_time} seconds...")

cobweb/launchers/launcher_pro.py

@@ -3,7 +3,7 @@ import threading
 
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
-from cobweb.utils import BloomFilter
+# from cobweb.utils import BloomFilter
 from cobweb.constant import DealModel, LogTemplate
 from .launcher import Launcher, check_pause
 
@@ -43,17 +43,18 @@ class LauncherPro(Launcher):
             self._db._client.incrby(key, count)
 
     def _get_seed(self) -> Seed:
-        spider_speed = self._db._client.get(self._speed_control_key)
-        if int(spider_speed or 0) > self._spider_max_count:
-            expire_time = self._db.ttl(self._speed_control_key)
-            if expire_time == -1:
-                self._db.delete(self._speed_control_key)
-            else:
-                logger.info(f"Too fast! Please wait {expire_time} seconds...")
-                time.sleep(expire_time / 2)
-            return None
+        if self._speed_control:
+            spider_speed = self._db._client.get(self._speed_control_key)
+            if int(spider_speed or 0) > self._spider_max_count:
+                expire_time = self._db.ttl(self._speed_control_key)
+                if expire_time == -1:
+                    self._db.delete(self._speed_control_key)
+                else:
+                    logger.info(f"Too fast! Please wait {expire_time} seconds...")
+                    time.sleep(expire_time / 2)
+                return None
         seed = self.__LAUNCHER_QUEUE__["todo"].pop()
-        if seed and not self._db.lock(self._speed_control_key, t=self._time_window):
+        if self._speed_control and seed and not self._db.lock(self._speed_control_key, t=self._time_window):
             self._db._client.incrby(self._speed_control_key, 1)
         return seed
 

cobweb/setting.py

@@ -65,6 +65,8 @@ TIME_WINDOW = 60   # 频控固定时间窗口（秒）
 # 任务模式
 TASK_MODEL = 0  # 0:单次，1:常驻
 
+# 流控措施
+SPEED_CONTROL = 1  # 0:关闭，1:开启
 
 # bloom过滤器
 CAPACITY = 100000000

{cobweb_launcher-1.2.50.dist-info → cobweb_launcher-1.2.52.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.50
+Version: 1.2.52
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.50.dist-info → cobweb_launcher-1.2.52.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=47HZsw40HLpsmOmvij1lyQALPQQCN_tWlKZ0wbn2MtM,2216
+cobweb/setting.py,sha256=JHRrVQL22iidkNe8seQTuxK3hlTdtnvb6VBVygpMs5I,2272
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/basic.py,sha256=Z56SSLB3I2IGHWCCcSy0Qbfzj8Qbg_po3gP32q1jh4k,7741
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
@@ -17,14 +17,14 @@ cobweb/crawlers/crawler.py,sha256=pePDGroD6JJAht5QTU51L7MkFIY4ob9TKpznZ2wUmsw,89
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb/db/redis_db.py,sha256=fumNZJiio-uQqRcSrymx8eJ1PqsdOwITe_Y-9JOXxrQ,4298
+cobweb/db/redis_db.py,sha256=mEWTBWSBXOhf3qxU9fdXYrhNJ44O8pS9tvqEAvZNhoI,5156
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
-cobweb/launchers/launcher.py,sha256=sPts-xlgxoeIfl1fn1XR2XVZxLzt7He9xrYDfTHRAGo,7029
+cobweb/launchers/launcher.py,sha256=_qC3EnD_QxQRP_RBNHHJ8HLfn_0FED1jUF_lyas5Hss,7081
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_api.py,sha256=YFqCTRvKn6icBLWTR1VxkU0WEIte2F7fv_LgPkifqdo,7885
-cobweb/launchers/launcher_pro.py,sha256=B5FdxvuENRL3XrMl74ENdP1uNgnZOaYCUUfBfM0t3io,7842
+cobweb/launchers/launcher_api.py,sha256=7K7Kl3dk7Ung9iRBwhiMrALEJywcR66ie5RIkLQEM-Y,7909
+cobweb/launchers/launcher_pro.py,sha256=QMumHUTi6IudO5mw9SnwlmP8k_BJ50cpNjYjoQv-_YY,7936
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -37,8 +37,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.50.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.50.dist-info/METADATA,sha256=uOO7AE9213AnlsYh-hhW2lCT_3-v9vaA7gmgiC1diuA,6510
-cobweb_launcher-1.2.50.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.50.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.50.dist-info/RECORD,,
+cobweb_launcher-1.2.52.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.52.dist-info/METADATA,sha256=lUniniJ1G9qDV9zA4XtI2p5bZPwDkMI6fKPSGnWjb_g,6510
+cobweb_launcher-1.2.52.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.52.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.52.dist-info/RECORD,,