cobweb-launcher 1.2.6__py3-none-any.whl → 1.2.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (7) hide show
  1. cobweb/launchers/launcher.py +2 -1
  2. cobweb/setting.py +1 -4
  3. {cobweb_launcher-1.2.6.dist-info → cobweb_launcher-1.2.7.dist-info}/METADATA +1 -1
  4. {cobweb_launcher-1.2.6.dist-info → cobweb_launcher-1.2.7.dist-info}/RECORD +7 -7
  5. {cobweb_launcher-1.2.6.dist-info → cobweb_launcher-1.2.7.dist-info}/LICENSE +0 -0
  6. {cobweb_launcher-1.2.6.dist-info → cobweb_launcher-1.2.7.dist-info}/WHEEL +0 -0
  7. {cobweb_launcher-1.2.6.dist-info → cobweb_launcher-1.2.7.dist-info}/top_level.txt +0 -0
cobweb/launchers/launcher.py CHANGED
@@ -4,7 +4,7 @@ import threading
4
4
  import importlib
5
5
 
6
6
  from cobweb import setting
7
- from cobweb.base import Seed, Queue
7
+ from cobweb.base import Seed, Queue, logger
8
8
  from cobweb.utils.tools import dynamic_load_class
9
9
 
10
10
 
@@ -138,6 +138,7 @@ class Launcher(threading.Thread):
138
138
  def _remove_doing_seeds(self, seeds):
139
139
  for seed in seeds:
140
140
  self.__DOING__.pop(seed, None)
141
+ logger.info("remove %s seeds from __DOING__" % len(seeds))
141
142
 
142
143
  def _execute(self):
143
144
  for func_name in self.__LAUNCHER_FUNC__:
cobweb/setting.py CHANGED
@@ -26,9 +26,6 @@ OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
26
26
  OSS_CHUNK_SIZE = 10 * 1024 ** 2
27
27
  OSS_MIN_UPLOAD_SIZE = 1024
28
28
 
29
- # message
30
- MESSAGE = ""
31
-
32
29
 
33
30
  # 采集器选择
34
31
  CRAWLER = "cobweb.crawlers.Crawler"
@@ -43,7 +40,7 @@ BEFORE_SCHEDULER_WAIT_SECONDS = 60 # 调度前等待时间,只作用于单次
43
40
  SCHEDULER_WAIT_SECONDS = 15 # 调度等待时间
44
41
  TODO_QUEUE_FULL_WAIT_SECONDS = 5 # todo队列已满时等待时间
45
42
  NEW_QUEUE_WAIT_SECONDS = 30 # new队列等待时间
46
- DONE_QUEUE_WAIT_SECONDS = 15 # done队列等待时间
43
+ DONE_QUEUE_WAIT_SECONDS = 5 # done队列等待时间
47
44
  UPLOAD_QUEUE_WAIT_SECONDS = 15 # upload队列等待时间
48
45
  SEED_RESET_SECONDS = 30 # 种子重制时间
49
46
 
{cobweb_launcher-1.2.6.dist-info → cobweb_launcher-1.2.7.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.6
3
+ Version: 1.2.7
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb_launcher-1.2.6.dist-info → cobweb_launcher-1.2.7.dist-info}/RECORD RENAMED
@@ -1,6 +1,6 @@
1
1
  cobweb/__init__.py,sha256=uMHyf4Fekbyw2xBCbkA8R0LwCpBJf5p_7pWbh60ZWYk,83
2
2
  cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
3
- cobweb/setting.py,sha256=bKM2J-yjGHeQn1Yf-yZ3cay_FTCPaokgOYSC6GPWwM4,2015
3
+ cobweb/setting.py,sha256=Wev0clo4ZETI5cRvBnzTWnJWo0Nowv_uvNCqlzYPSiE,1990
4
4
  cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
5
5
  cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
6
6
  cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -18,7 +18,7 @@ cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
18
18
  cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
19
19
  cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
20
20
  cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
21
- cobweb/launchers/launcher.py,sha256=0YhJtmN048MgTZ7SAUWvaKJEPXrNx0p4dkoTopsyjzc,5466
21
+ cobweb/launchers/launcher.py,sha256=xGoELgImXrBgQCuHm8Wj7_HmJoaH7osB0ZvKHiSJ0_0,5541
22
22
  cobweb/launchers/launcher_air.py,sha256=zHVEJqQCxYU1WDnqQzzEHbEXasR1GmKevujQkCfFt5o,2947
23
23
  cobweb/launchers/launcher_pro.py,sha256=qCmobbi1jhI09cVRjaYGxyTHwBSvARiHxF1p37b5iaE,8107
24
24
  cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
@@ -30,8 +30,8 @@ cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXt
30
30
  cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
31
31
  cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
32
32
  cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
33
- cobweb_launcher-1.2.6.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
34
- cobweb_launcher-1.2.6.dist-info/METADATA,sha256=q_ihYp0LYgZRMdvzJ2gmTYG0HgeoOAnRjOEj5FAUp34,6489
35
- cobweb_launcher-1.2.6.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
36
- cobweb_launcher-1.2.6.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
37
- cobweb_launcher-1.2.6.dist-info/RECORD,,
33
+ cobweb_launcher-1.2.7.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
34
+ cobweb_launcher-1.2.7.dist-info/METADATA,sha256=S_GUTo_QpPqHjgSC_M3j3qn1zJL90XAAQUB4Cm8JU48,6489
35
+ cobweb_launcher-1.2.7.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
36
+ cobweb_launcher-1.2.7.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
37
+ cobweb_launcher-1.2.7.dist-info/RECORD,,