cobweb-launcher 3.1.1__py3-none-any.whl → 3.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. cobweb/launchers/distributor.py +2 -2
  2. cobweb/launchers/launcher.py +2 -1
  3. cobweb/schedulers/scheduler_with_redis.py +1 -1
  4. {cobweb_launcher-3.1.1.dist-info → cobweb_launcher-3.1.3.dist-info}/METADATA +1 -1
  5. {cobweb_launcher-3.1.1.dist-info → cobweb_launcher-3.1.3.dist-info}/RECORD +8 -8
  6. {cobweb_launcher-3.1.1.dist-info → cobweb_launcher-3.1.3.dist-info}/LICENSE +0 -0
  7. {cobweb_launcher-3.1.1.dist-info → cobweb_launcher-3.1.3.dist-info}/WHEEL +0 -0
  8. {cobweb_launcher-3.1.1.dist-info → cobweb_launcher-3.1.3.dist-info}/top_level.txt +0 -0
cobweb/launchers/distributor.py CHANGED
@@ -59,13 +59,13 @@ class Distributor(threading.Thread):
59
59
  def distribute(self, item, seed, _id: int):
60
60
  if isinstance(item, Request):
61
61
  seed.params.start_time = time.time()
62
- self.process(item=seed, seed=seed, callback=self.Crawler.download, _id=1)
62
+ self.process(item=item, seed=seed, callback=self.Crawler.download, _id=1)
63
63
  elif isinstance(item, Response):
64
64
  if _id == 2:
65
65
  raise TypeError("parse function can't yield a Response instance")
66
66
  dot = isinstance(item.response, Res)
67
67
  self.spider_logging(seed, item, dot=dot)
68
- self.process(item=seed, seed=seed, callback=self.Crawler.parse, _id=2)
68
+ self.process(item=item, seed=seed, callback=self.Crawler.parse, _id=2)
69
69
  elif isinstance(item, BaseItem):
70
70
  self.upload.push(item)
71
71
  elif isinstance(item, Seed):
cobweb/launchers/launcher.py CHANGED
@@ -7,7 +7,8 @@ import importlib
7
7
  from cobweb import setting
8
8
  from cobweb.base import Seed, Queue, logger
9
9
  from cobweb.utils.tools import dynamic_load_class
10
- from cobweb.launchers import Distributor, Uploader
10
+ from cobweb.launchers.distributor import Distributor
11
+ from cobweb.launchers.uploader import Uploader
11
12
 
12
13
 
13
14
  class Launcher:
cobweb/schedulers/scheduler_with_redis.py CHANGED
@@ -7,7 +7,7 @@ from cobweb.utils import check_pause
7
7
  from cobweb.base import Queue, Seed, logger
8
8
  from cobweb.constant import LogTemplate
9
9
  from .scheduler import Scheduler
10
- use_api = bool(int(os.getenv("REDIS_API")))
10
+ use_api = bool(int(os.getenv("REDIS_API", 0)))
11
11
 
12
12
 
13
13
  class RedisScheduler(Scheduler):
{cobweb_launcher-3.1.1.dist-info → cobweb_launcher-3.1.3.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.1
3
+ Version: 3.1.3
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb_launcher-3.1.1.dist-info → cobweb_launcher-3.1.3.dist-info}/RECORD RENAMED
@@ -16,8 +16,8 @@ cobweb/db/redis_db.py,sha256=rK6PnKLLVTeg6HMGVtlYl4_b5og3aO2J5jZrqU2Aoso,7721
16
16
  cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
17
17
  cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
18
18
  cobweb/launchers/__init__.py,sha256=XEW4hmBMPKp2bRkHBxxlplXwvKdlLVYBFg3etjtbRXo,222
19
- cobweb/launchers/distributor.py,sha256=Br5G6Jn6aJeaAX9y3yBfNVRvq2-kIIz9LsjRCXLmzGg,6447
20
- cobweb/launchers/launcher.py,sha256=WTyv5bgKE_VwZXuWia_F26fH2zi6mps0wsnC16bks9k,5253
19
+ cobweb/launchers/distributor.py,sha256=WWLyDaSGMWQWKfZX9guUzIZEwAX9FM5pqOUP3WDfEak,6447
20
+ cobweb/launchers/launcher.py,sha256=rDT7OQX3wzaS_MsbrkeWDH5IHpN5GE_Oy4tUkvnovKM,5302
21
21
  cobweb/launchers/uploader.py,sha256=2zAYMEpfUYo8zCzH6Jhwkj9Y4_aJYVNnx0D5H8srKW4,1806
22
22
  cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
23
23
  cobweb/pipelines/pipeline.py,sha256=qwoOYMhlAB-MnEmMNpNeauTHoRTOr2wyBDYS4MF6B1c,261
@@ -27,15 +27,15 @@ cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0
27
27
  cobweb/schedulers/launcher_air.py,sha256=qdcxq41I9zN5snEpMUUNEQNTtUiZM1Hw_3N9zu4PuAs,3058
28
28
  cobweb/schedulers/launcher_api.py,sha256=3-A6k3Igvi-xnvP9M_3NkJKUweDJ_pY10ZrHClteD-g,8628
29
29
  cobweb/schedulers/scheduler.py,sha256=mN9XvaOCzNnBWQfzslTIM860ZGq2gyLtxpbVUd0Slqs,2240
30
- cobweb/schedulers/scheduler_with_redis.py,sha256=pCimXqqpJ2xI1fs-6Ecy2vs9vHC-PPFX4IBMLDtpbho,6394
30
+ cobweb/schedulers/scheduler_with_redis.py,sha256=LmfQ_uwpXxwvG_i52pDl93T_fxqx3eWg4M2XgDgvmGQ,6397
31
31
  cobweb/utils/__init__.py,sha256=8Bu5iZrIOUMS4jv4hi0inRPtscf6MK0ZFa7gQ7ZFoqw,145
32
32
  cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
33
33
  cobweb/utils/decorators.py,sha256=066JCY_RNMr2mXkhEv8XTtOOKkv9CFiBm0ZNCcC-2ag,1131
34
34
  cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
35
35
  cobweb/utils/oss.py,sha256=ktfwMGnq5FMzOkUUS3nNXI7pTdPNinudH3YRJ0nMhoU,3985
36
36
  cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
37
- cobweb_launcher-3.1.1.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
38
- cobweb_launcher-3.1.1.dist-info/METADATA,sha256=nYc7GKMsOuvDRqPRxsakSM9VbM3TBH1Xs7yqBosl_8g,6509
39
- cobweb_launcher-3.1.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
40
- cobweb_launcher-3.1.1.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
41
- cobweb_launcher-3.1.1.dist-info/RECORD,,
37
+ cobweb_launcher-3.1.3.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
38
+ cobweb_launcher-3.1.3.dist-info/METADATA,sha256=HixKAsXNcFFZapClC_17LAjD1OLUSgQDWa-SFZNPX9c,6509
39
+ cobweb_launcher-3.1.3.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
40
+ cobweb_launcher-3.1.3.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
41
+ cobweb_launcher-3.1.3.dist-info/RECORD,,