cobweb-launcher 1.2.54__py3-none-any.whl → 1.2.56__py3-none-any.whl

cobweb/db/redis_db.py

@@ -6,11 +6,7 @@ class RedisDB:
 
     def __init__(self, **kwargs):
         redis_config = kwargs or setting.REDIS_CONFIG
-        # pool = redis.ConnectionPool(**redis_config)
-        self.pool = redis.ConnectionPool(
-            max_connections=25,
-            **redis_config
-        )
+        self.pool = redis.ConnectionPool(**redis_config)
         
     def get_connection(self):
         return redis.StrictRedis(connection_pool=self.pool)
@@ -147,6 +143,10 @@ class RedisDB:
         members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
         return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
 
+    # def get_member(self):
+    #     with self.get_connection() as client:
+    #         pipeline = client.pipeline()
+
     def done(self, keys: list, *args) -> list:
         lua_script = """
         for i, member in ipairs(ARGV) do

cobweb/launchers/launcher_pro.py

@@ -3,8 +3,7 @@ import threading
 
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
-# from cobweb.utils import BloomFilter
-from cobweb.constant import DealModel, LogTemplate
+from cobweb.constant import LogTemplate
 from .launcher import Launcher, check_pause
 
 
@@ -128,7 +127,7 @@ class LauncherPro(Launcher):
             refresh_time = int(time.time())
             seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
             self._db.zadd(self._todo_key, item=seeds, xx=True)
-        time.sleep(15)
+        time.sleep(3)
 
     @check_pause
     def _delete(self):
@@ -175,7 +174,7 @@ class LauncherPro(Launcher):
                         logger.info("Done! ready to close thread...")
                         self._stop.set()
 
-                    elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
+                    elif self._db.zcard(self._todo_key):
                         logger.info(f"Recovery {self.task} task run！")
                         self._pause.clear()
                         self._execute()
@@ -183,8 +182,14 @@ class LauncherPro(Launcher):
                         logger.info("pause! waiting for resume...")
                 elif check_emtpy_times > 2:
                     self.__DOING__ = {}
-                    if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
+                    seed_count = self._db.zcard(self._todo_key)
+                    logger.info(f"队列剩余种子数:{seed_count}")
+                    if not seed_count:
+                        logger.info("Done! pause set...")
                         self._pause.set()
+                    else:
+                        self._pause.clear()
+
                 else:
                     logger.info(
                         "check whether the task is complete, "

cobweb/setting.py

@@ -42,7 +42,7 @@ TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
 DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
 UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
-SEED_RESET_SECONDS = 30   # 种子重制时间
+SEED_RESET_SECONDS = 600   # 种子重制时间
 
 
 # Launcher 队列长度

{cobweb_launcher-1.2.54.dist-info → cobweb_launcher-1.2.56.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.54
+Version: 1.2.56
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.54.dist-info → cobweb_launcher-1.2.56.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=JHRrVQL22iidkNe8seQTuxK3hlTdtnvb6VBVygpMs5I,2272
+cobweb/setting.py,sha256=LlWZeZLVMeCd9NRCPkFDf_DK0IUua1L2U_zYFPJpn_k,2273
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/basic.py,sha256=Z56SSLB3I2IGHWCCcSy0Qbfzj8Qbg_po3gP32q1jh4k,7741
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
@@ -17,14 +17,14 @@ cobweb/crawlers/crawler.py,sha256=pePDGroD6JJAht5QTU51L7MkFIY4ob9TKpznZ2wUmsw,89
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb/db/redis_db.py,sha256=DG-8l5DKRTs2ln1g9SM6gBMkmmjAcCM7RHL84eIuH60,5385
+cobweb/db/redis_db.py,sha256=ODV_p00TOQb7hvzR1fxFFOFryPL_VxkezdCcFgVKb0s,5397
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
 cobweb/launchers/launcher.py,sha256=y1a72sANV-oJVpy_BI7BwKy7fY4AnpHqJK3t6czoCts,7896
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
 cobweb/launchers/launcher_api.py,sha256=7K7Kl3dk7Ung9iRBwhiMrALEJywcR66ie5RIkLQEM-Y,7909
-cobweb/launchers/launcher_pro.py,sha256=QCmOzlIDvdfdfZVK7rQN6PYou9Vedepie1aqqeHlI2w,7912
+cobweb/launchers/launcher_pro.py,sha256=_KK0KlrzDaQZOdTQJoPZcBsoAiWF13CSbvoepahXIpU,8054
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -37,8 +37,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.54.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.54.dist-info/METADATA,sha256=n9jaFGNujOnCTREmaWA7yM66RKuj5VK1aAvQ_EJRqiU,6510
-cobweb_launcher-1.2.54.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.54.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.54.dist-info/RECORD,,
+cobweb_launcher-1.2.56.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.56.dist-info/METADATA,sha256=vMZOi2X8HJQHxiaTroTzoLl6HNT95-bq5BpiS9o6nNM,6510
+cobweb_launcher-1.2.56.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.56.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.56.dist-info/RECORD,,