cobweb-launcher 0.1.18__py3-none-any.whl → 0.1.19__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (8) hide show
  1. cobweb/constant.py +1 -0
  2. cobweb/equip/single/models.py +4 -4
  3. cobweb/task.py +1 -0
  4. {cobweb_launcher-0.1.18.dist-info → cobweb_launcher-0.1.19.dist-info}/METADATA +1 -1
  5. {cobweb_launcher-0.1.18.dist-info → cobweb_launcher-0.1.19.dist-info}/RECORD +8 -8
  6. {cobweb_launcher-0.1.18.dist-info → cobweb_launcher-0.1.19.dist-info}/LICENSE +0 -0
  7. {cobweb_launcher-0.1.18.dist-info → cobweb_launcher-0.1.19.dist-info}/WHEEL +0 -0
  8. {cobweb_launcher-0.1.18.dist-info → cobweb_launcher-0.1.19.dist-info}/top_level.txt +0 -0
cobweb/constant.py CHANGED
@@ -25,3 +25,4 @@ class Setting:
25
25
  SCHEDULER_WAIT_TIME = None
26
26
  SCHEDULER_BLOCK_TIME = None
27
27
  SPIDER_WAIT_TIME = None
28
+ SPIDER_SLEEP_TIME = None
cobweb/equip/single/models.py CHANGED
@@ -33,11 +33,11 @@ class Scheduler:
33
33
  while not stop.is_set():
34
34
 
35
35
  if not ready_seed_length():
36
- time.sleep(5)
36
+ time.sleep(Setting.SCHEDULER_WAIT_TIME)
37
37
  continue
38
38
 
39
39
  if self.queue.length >= self.length:
40
- time.sleep(3)
40
+ time.sleep(Setting.SCHEDULER_BLOCK_TIME)
41
41
  continue
42
42
 
43
43
  seeds = get_seed(self.length)
@@ -59,7 +59,7 @@ class Spider:
59
59
  seed = self.queue.pop()
60
60
 
61
61
  if not seed:
62
- time.sleep(3)
62
+ time.sleep(Setting.SPIDER_WAIT_TIME)
63
63
  continue
64
64
 
65
65
  elif seed._retry >= self.max_retries:
@@ -122,7 +122,7 @@ class Spider:
122
122
  log.info(f"{str(seed)} -> {str(e)}")
123
123
  finally:
124
124
  self.spider_in_progress.pop()
125
- time.sleep(Setting.SPIDER_RUN_TIME)
125
+ time.sleep(Setting.SPIDER_SLEEP_TIME)
126
126
  log.info(f"close thread: spider")
127
127
 
128
128
 
cobweb/task.py CHANGED
@@ -11,6 +11,7 @@ def init_task_env():
11
11
  Setting.SCHEDULER_WAIT_TIME = float(os.getenv("SCHEDULER_WAIT_TIME", 5))
12
12
  Setting.SCHEDULER_BLOCK_TIME = float(os.getenv("SCHEDULER_BLOCK_TIME", 3))
13
13
  Setting.SPIDER_WAIT_TIME = float(os.getenv("SPIDER_WAIT_TIME", 3))
14
+ Setting.SPIDER_SLEEP_TIME = float(os.getenv("SPIDER_SLEEP_TIME", 0.5))
14
15
 
15
16
 
16
17
  class Task:
{cobweb_launcher-0.1.18.dist-info → cobweb_launcher-0.1.19.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 0.1.18
3
+ Version: 0.1.19
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb_launcher-0.1.18.dist-info → cobweb_launcher-0.1.19.dist-info}/RECORD RENAMED
@@ -1,11 +1,11 @@
1
1
  cobweb/__init__.py,sha256=zVims_awYxKNqULuKSjFXdZAaDJaqzFBJJo_SqeakMs,286
2
2
  cobweb/bbb.py,sha256=zKeCeBVFQfhEv6M8CCktUTM7tXDZmAu6ZN0-ET44pUY,5707
3
- cobweb/constant.py,sha256=ytkGczukfFU9bx4ricEQy7eY3Odwc4Xmrps9PwuGiuo,553
3
+ cobweb/constant.py,sha256=C3X-4BLSyG830Z661rq67yOkAyKBOh04K88duAvMgvU,582
4
4
  cobweb/decorators.py,sha256=eYQI9rddPVJihAlomLTmbtQhIOzPw8dCrOFpxAq2pLY,318
5
5
  cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
6
6
  cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
7
7
  cobweb/setting.py,sha256=UAu_dLuIFYO98MxtlZ5sZqJcwKAUKq4Bu4KoKlV50Mc,288
8
- cobweb/task.py,sha256=xoZHZEH2FFmc0tCTZS0pLhEKyXHroECpQBKw40y4qQ0,1931
8
+ cobweb/task.py,sha256=SyWC43C7hqQAqH-1HECXEEgp_6L6lwDhYe1kZNnHUvA,2006
9
9
  cobweb/utils.py,sha256=ivmRqJJNtwdOKYT4G7qQCWnL8ar9c-shxeDZzGB2E9c,2651
10
10
  cobweb/db/__init__.py,sha256=jC-uOThYLtiDUG6cTJRkDITgOzR4nIOeaZQeZhuk-v0,139
11
11
  cobweb/db/oss_db.py,sha256=59HwMMfoGUr6l_BI4p84YHYoQwEn22cCM_1GZK34Uo4,4244
@@ -30,13 +30,13 @@ cobweb/equip/distributed/launcher.py,sha256=1LzxibGXWR20XpXawakiRpEMaa9yfaj2rFSK
30
30
  cobweb/equip/distributed/models.py,sha256=qTGzxLdb2arsZSZK2HE4-MrqhraUhc2Ol5wBvlv_aWA,5008
31
31
  cobweb/equip/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
32
32
  cobweb/equip/single/launcher.py,sha256=KRsw7yxklvFM85cel-EyLsNPLyrC9Hd26BMSx6-4Hac,6785
33
- cobweb/equip/single/models.py,sha256=pWALuEat5v9cNAvJ_PaAlDcIHUWuu38EQl0KFcpSJXA,5344
33
+ cobweb/equip/single/models.py,sha256=VwaGgZXq72a48zsZja100Lj2GBHF1GkYTddTi1R-RxM,5422
34
34
  cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
35
35
  cobweb/single/launcher.py,sha256=IoJbn87j7t7Pib_FxoWZmmX8asXOqNGb-9ospw6EYJI,7302
36
36
  cobweb/single/models.py,sha256=wIEV35666lxdzqjDqBHPjjh-r6zD0x24rtQYz7d4Oxw,4332
37
37
  cobweb/single/nest.py,sha256=49K6KQ934INfPrWQsrq9rIFpQauLbLGOFbDaHvoQzOk,5015
38
- cobweb_launcher-0.1.18.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
39
- cobweb_launcher-0.1.18.dist-info/METADATA,sha256=QPYACYelA777bRzaswA-ETRqKIcf5crC_BKYE1wyrEc,1220
40
- cobweb_launcher-0.1.18.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
41
- cobweb_launcher-0.1.18.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
42
- cobweb_launcher-0.1.18.dist-info/RECORD,,
38
+ cobweb_launcher-0.1.19.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
39
+ cobweb_launcher-0.1.19.dist-info/METADATA,sha256=SRjTvi2-SkwmmHwhwEy9XeLPmjsbG9XYqGi651rJD20,1220
40
+ cobweb_launcher-0.1.19.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
41
+ cobweb_launcher-0.1.19.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
42
+ cobweb_launcher-0.1.19.dist-info/RECORD,,