cobweb-launcher 3.1.21__py3-none-any.whl → 3.1.22__py3-none-any.whl

cobweb/schedulers/scheduler_with_redis.py

@@ -40,7 +40,8 @@ class RedisScheduler(Scheduler):
         while not self.stop.is_set():
             if self.db.lock(self.reset_lock_key, t=60):
 
-                _min = -int(time.time()) + self.seed_reset_seconds
+                # _min = -int(time.time()) + self.seed_reset_seconds
+                _min = -int(time.time()) + 3600
                 self.db.members(self.todo_key, 0, _min=_min, _max="(0")
                 self.db.delete(self.reset_lock_key)
 
@@ -59,12 +60,6 @@ class RedisScheduler(Scheduler):
             time.sleep(self.todo_queue_full_wait_seconds)
             return
 
-        count = self.get_working_items_count()
-        if count > 500:
-            time.sleep(self.todo_queue_full_wait_seconds)
-            logger.info(f"任务挤压, 内存任务数：{count}")
-            return
-
         members = self.db.members(
             self.todo_key, int(time.time()),
             count=self.todo_queue_size,
@@ -79,9 +74,9 @@ class RedisScheduler(Scheduler):
             item_info[seed.to_string] = seed.params.priority
             seeds.append(seed)
 
-        self.set_working_items(item_info)
+        # self.set_working_items(item_info)
         self.todo.push(seeds)
-        time.sleep(1)
+        # time.sleep(1)
 
     @check_pause
     def insert(self):

{cobweb_launcher-3.1.21.dist-info → cobweb_launcher-3.1.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.21
+Version: 3.1.22
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-3.1.21.dist-info → cobweb_launcher-3.1.22.dist-info}/RECORD

@@ -25,15 +25,15 @@ cobweb/pipelines/pipeline_csv.py,sha256=TFqxqgVUqkBF6Jott4zd6fvCSxzG67lpafRQtXPw
 cobweb/pipelines/pipeline_loghub.py,sha256=zwIa_pcWBB2UNGd32Cu-i1jKGNruTbo2STdxl1WGwZ0,1829
 cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
 cobweb/schedulers/scheduler.py,sha256=qb_u3tJELp4zRGOyT8OsNWndKDknhQslnKnjUlwEpiE,2943
-cobweb/schedulers/scheduler_with_redis.py,sha256=8-uVPM2RAqd1Azcwd1Pm5mXHekxzRTIpHyGyPjOYtDk,6611
+cobweb/schedulers/scheduler_with_redis.py,sha256=i7EfjXY4p5BJrZfCNxyUqBqIcyg1ZICjm7gbxrhHbN0,6448
 cobweb/utils/__init__.py,sha256=TRFJyyBjaQH_sejU6G_msOeHpjc3ZXU0dUOO5GQfknM,171
 cobweb/utils/bloom.py,sha256=A8xqtHXp7jgRoBuUlpovmq8lhU5y7IEF0FOCjfQDb6s,1855
 cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,1140
 cobweb/utils/dotting.py,sha256=L-jGSApdnFIP4jUWH6p5qIme0aJ1vyDrxAx8wOJWvcs,1960
 cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
 cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
-cobweb_launcher-3.1.21.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.21.dist-info/METADATA,sha256=ykVyNpvoprcTRZvOUxKtTCM8jDoqZuy_ewBzjY-v8u0,5998
-cobweb_launcher-3.1.21.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.21.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.21.dist-info/RECORD,,
+cobweb_launcher-3.1.22.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.22.dist-info/METADATA,sha256=L5ozmjklF83tH1LhHCgFp2U-rRBzEP_sj6x8WSZda5s,5998
+cobweb_launcher-3.1.22.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.22.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.22.dist-info/RECORD,,