cobweb-launcher 3.1.19__py3-none-any.whl → 3.1.21__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. cobweb/schedulers/scheduler_with_redis.py +7 -0
  2. {cobweb_launcher-3.1.19.dist-info → cobweb_launcher-3.1.21.dist-info}/METADATA +1 -1
  3. {cobweb_launcher-3.1.19.dist-info → cobweb_launcher-3.1.21.dist-info}/RECORD +6 -6
  4. {cobweb_launcher-3.1.19.dist-info → cobweb_launcher-3.1.21.dist-info}/LICENSE +0 -0
  5. {cobweb_launcher-3.1.19.dist-info → cobweb_launcher-3.1.21.dist-info}/WHEEL +0 -0
  6. {cobweb_launcher-3.1.19.dist-info → cobweb_launcher-3.1.21.dist-info}/top_level.txt +0 -0
cobweb/schedulers/scheduler_with_redis.py CHANGED
@@ -59,6 +59,12 @@ class RedisScheduler(Scheduler):
59
59
  time.sleep(self.todo_queue_full_wait_seconds)
60
60
  return
61
61
 
62
+ count = self.get_working_items_count()
63
+ if count > 500:
64
+ time.sleep(self.todo_queue_full_wait_seconds)
65
+ logger.info(f"任务挤压, 内存任务数：{count}")
66
+ return
67
+
62
68
  members = self.db.members(
63
69
  self.todo_key, int(time.time()),
64
70
  count=self.todo_queue_size,
@@ -75,6 +81,7 @@ class RedisScheduler(Scheduler):
75
81
 
76
82
  self.set_working_items(item_info)
77
83
  self.todo.push(seeds)
84
+ time.sleep(1)
78
85
 
79
86
  @check_pause
80
87
  def insert(self):
{cobweb_launcher-3.1.19.dist-info → cobweb_launcher-3.1.21.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.19
3
+ Version: 3.1.21
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb_launcher-3.1.19.dist-info → cobweb_launcher-3.1.21.dist-info}/RECORD RENAMED
@@ -25,15 +25,15 @@ cobweb/pipelines/pipeline_csv.py,sha256=TFqxqgVUqkBF6Jott4zd6fvCSxzG67lpafRQtXPw
25
25
  cobweb/pipelines/pipeline_loghub.py,sha256=zwIa_pcWBB2UNGd32Cu-i1jKGNruTbo2STdxl1WGwZ0,1829
26
26
  cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
27
27
  cobweb/schedulers/scheduler.py,sha256=qb_u3tJELp4zRGOyT8OsNWndKDknhQslnKnjUlwEpiE,2943
28
- cobweb/schedulers/scheduler_with_redis.py,sha256=4adyaLBZe7-oaAQbCa99a0NDjwcIA3wzxaOqW8GrBrM,6372
28
+ cobweb/schedulers/scheduler_with_redis.py,sha256=8-uVPM2RAqd1Azcwd1Pm5mXHekxzRTIpHyGyPjOYtDk,6611
29
29
  cobweb/utils/__init__.py,sha256=TRFJyyBjaQH_sejU6G_msOeHpjc3ZXU0dUOO5GQfknM,171
30
30
  cobweb/utils/bloom.py,sha256=A8xqtHXp7jgRoBuUlpovmq8lhU5y7IEF0FOCjfQDb6s,1855
31
31
  cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,1140
32
32
  cobweb/utils/dotting.py,sha256=L-jGSApdnFIP4jUWH6p5qIme0aJ1vyDrxAx8wOJWvcs,1960
33
33
  cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
34
34
  cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
35
- cobweb_launcher-3.1.19.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
36
- cobweb_launcher-3.1.19.dist-info/METADATA,sha256=yGiXxEUmI_j5vz3KDorlcubne0C6eCLj8VM5O3kr4fw,5998
37
- cobweb_launcher-3.1.19.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
38
- cobweb_launcher-3.1.19.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
39
- cobweb_launcher-3.1.19.dist-info/RECORD,,
35
+ cobweb_launcher-3.1.21.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
36
+ cobweb_launcher-3.1.21.dist-info/METADATA,sha256=ykVyNpvoprcTRZvOUxKtTCM8jDoqZuy_ewBzjY-v8u0,5998
37
+ cobweb_launcher-3.1.21.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
38
+ cobweb_launcher-3.1.21.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
39
+ cobweb_launcher-3.1.21.dist-info/RECORD,,