cobweb-launcher 3.1.6__py3-none-any.whl → 3.1.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. cobweb/launchers/distributor.py +1 -1
  2. cobweb/launchers/launcher.py +1 -0
  3. cobweb/launchers/uploader.py +0 -2
  4. cobweb/utils/dotting.py +3 -2
  5. {cobweb_launcher-3.1.6.dist-info → cobweb_launcher-3.1.8.dist-info}/METADATA +1 -1
  6. {cobweb_launcher-3.1.6.dist-info → cobweb_launcher-3.1.8.dist-info}/RECORD +9 -9
  7. {cobweb_launcher-3.1.6.dist-info → cobweb_launcher-3.1.8.dist-info}/LICENSE +0 -0
  8. {cobweb_launcher-3.1.6.dist-info → cobweb_launcher-3.1.8.dist-info}/WHEEL +0 -0
  9. {cobweb_launcher-3.1.6.dist-info → cobweb_launcher-3.1.8.dist-info}/top_level.txt +0 -0
cobweb/launchers/distributor.py CHANGED
@@ -44,7 +44,7 @@ class Distributor(threading.Thread):
44
44
  self.thread_num = setting.SPIDER_THREAD_NUM
45
45
  self.max_retries = setting.SPIDER_MAX_RETRIES
46
46
  self.record_failed = setting.RECORD_FAILED_SPIDER
47
- self.loghub_dot = LoghubDot() # todo: 解偶
47
+ self.loghub_dot = LoghubDot(stop=stop) # todo: 解偶
48
48
 
49
49
  self.Crawler = SpiderCrawler
50
50
 
cobweb/launchers/launcher.py CHANGED
@@ -128,6 +128,7 @@ class Launcher:
128
128
  self.__WORKER_THREAD__[name] = worker_thread
129
129
  worker_thread.start()
130
130
  time.sleep(3)
131
+ logger.info("monitor thread close ...")
131
132
 
132
133
  def start(self):
133
134
  self._pause.is_set()
cobweb/launchers/uploader.py CHANGED
@@ -57,8 +57,6 @@ class Uploader(threading.Thread):
57
57
  if seeds:
58
58
  self.done.push(seeds)
59
59
 
60
- logger.info("upload pipeline close!")
61
-
62
60
  def run(self):
63
61
  self.register(self.upload_data, tag="Uploader")
64
62
 
cobweb/utils/dotting.py CHANGED
@@ -9,7 +9,8 @@ from cobweb import setting
9
9
 
10
10
  class LoghubDot:
11
11
 
12
- def __init__(self):
12
+ def __init__(self, stop):
13
+ self.stop = stop
13
14
  self.client = LogClient(**setting.LOGHUB_CONFIG)
14
15
  self.queue = Queue()
15
16
 
@@ -27,7 +28,7 @@ class LoghubDot:
27
28
  self.queue.push((topic, log_item), direct_insertion=True)
28
29
 
29
30
  def build_run(self):
30
- while True:
31
+ while not self.stop.is_set():
31
32
  start_time = int(time.time())
32
33
  while True:
33
34
  cost_time = int(time.time()) - start_time
{cobweb_launcher-3.1.6.dist-info → cobweb_launcher-3.1.8.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.6
3
+ Version: 3.1.8
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb_launcher-3.1.6.dist-info → cobweb_launcher-3.1.8.dist-info}/RECORD RENAMED
@@ -16,9 +16,9 @@ cobweb/db/redis_db.py,sha256=rK6PnKLLVTeg6HMGVtlYl4_b5og3aO2J5jZrqU2Aoso,7721
16
16
  cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
17
17
  cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
18
18
  cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
19
- cobweb/launchers/distributor.py,sha256=-tr3c3O3QBV9UYH1optdBfq76_o83RjII5YKiIYl33w,6420
20
- cobweb/launchers/launcher.py,sha256=rDT7OQX3wzaS_MsbrkeWDH5IHpN5GE_Oy4tUkvnovKM,5302
21
- cobweb/launchers/uploader.py,sha256=gJiKMVeoxxwrLsYPaMuXJN7wc5LFxXfjC4VVIlqqlqk,1818
19
+ cobweb/launchers/distributor.py,sha256=yZJ7d8EXZGukRr8SzcxXZu1z37VxKqBWE1apNW9T9uE,6429
20
+ cobweb/launchers/launcher.py,sha256=zP0l-5QXxoZauh2bnIuHUJ6L5QI0bbgXvhEN49Ff43g,5350
21
+ cobweb/launchers/uploader.py,sha256=ePQsGxoFw3Le4DgkoDGDqxrv8lGaYKCnGq2nuaWba9o,1771
22
22
  cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
23
23
  cobweb/pipelines/pipeline.py,sha256=qwoOYMhlAB-MnEmMNpNeauTHoRTOr2wyBDYS4MF6B1c,261
24
24
  cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -31,11 +31,11 @@ cobweb/schedulers/scheduler_with_redis.py,sha256=Qq8iMxbDWJpRml76NfcZyyB8q7tM2Et
31
31
  cobweb/utils/__init__.py,sha256=8Bu5iZrIOUMS4jv4hi0inRPtscf6MK0ZFa7gQ7ZFoqw,145
32
32
  cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
33
33
  cobweb/utils/decorators.py,sha256=066JCY_RNMr2mXkhEv8XTtOOKkv9CFiBm0ZNCcC-2ag,1131
34
- cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
34
+ cobweb/utils/dotting.py,sha256=x34WryNKu_o54EzVwAZByagjMTXF0ZP0VYKdjfYEdSM,1833
35
35
  cobweb/utils/oss.py,sha256=ktfwMGnq5FMzOkUUS3nNXI7pTdPNinudH3YRJ0nMhoU,3985
36
36
  cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
37
- cobweb_launcher-3.1.6.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
38
- cobweb_launcher-3.1.6.dist-info/METADATA,sha256=1hnno3Ujgp1nUQP2VTHAMy-VyUFMx09qRYjo4O4Iz3E,6509
39
- cobweb_launcher-3.1.6.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
40
- cobweb_launcher-3.1.6.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
41
- cobweb_launcher-3.1.6.dist-info/RECORD,,
37
+ cobweb_launcher-3.1.8.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
38
+ cobweb_launcher-3.1.8.dist-info/METADATA,sha256=dHvMv0I_533y0KCp5RSVT87mqbznPgWakYtN3OeN3KQ,6509
39
+ cobweb_launcher-3.1.8.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
40
+ cobweb_launcher-3.1.8.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
41
+ cobweb_launcher-3.1.8.dist-info/RECORD,,