cobweb-launcher 3.1.17__tar.gz → 3.1.19__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/PKG-INFO +1 -1
  2. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/schedulers/scheduler.py +1 -1
  3. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/schedulers/scheduler_with_redis.py +3 -3
  4. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb_launcher.egg-info/PKG-INFO +1 -1
  5. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/setup.py +1 -1
  6. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/LICENSE +0 -0
  7. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/README.md +0 -0
  8. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/__init__.py +0 -0
  9. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/base/__init__.py +0 -0
  10. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/base/common_queue.py +0 -0
  11. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/base/item.py +0 -0
  12. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/base/logger.py +0 -0
  13. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/base/request.py +0 -0
  14. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/base/response.py +0 -0
  15. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/base/seed.py +0 -0
  16. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/constant.py +0 -0
  17. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/crawlers/__init__.py +0 -0
  18. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/crawlers/crawler.py +0 -0
  19. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/db/__init__.py +0 -0
  20. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/db/api_db.py +0 -0
  21. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/db/redis_db.py +0 -0
  22. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/exceptions/__init__.py +0 -0
  23. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/exceptions/oss_db_exception.py +0 -0
  24. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/launchers/__init__.py +0 -0
  25. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/launchers/distributor.py +0 -0
  26. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/launchers/launcher.py +0 -0
  27. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/launchers/uploader.py +0 -0
  28. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/pipelines/__init__.py +0 -0
  29. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/pipelines/pipeline.py +0 -0
  30. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/pipelines/pipeline_csv.py +0 -0
  31. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/pipelines/pipeline_loghub.py +0 -0
  32. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/schedulers/__init__.py +0 -0
  33. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/setting.py +0 -0
  34. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/utils/__init__.py +0 -0
  35. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/utils/bloom.py +0 -0
  36. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/utils/decorators.py +0 -0
  37. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/utils/dotting.py +0 -0
  38. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/utils/oss.py +0 -0
  39. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/utils/tools.py +0 -0
  40. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  41. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  42. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb_launcher.egg-info/requires.txt +0 -0
  43. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb_launcher.egg-info/top_level.txt +0 -0
  44. {cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/setup.cfg +0 -0
{cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.17
3
+ Version: 3.1.19
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/schedulers/scheduler.py RENAMED
@@ -57,7 +57,7 @@ class Scheduler(ABC, threading.Thread):
57
57
 
58
58
  def set_working_items(self, item_info: dict = None):
59
59
  if not item_info:
60
- return
60
+ return False
61
61
  with self.lock:
62
62
  self.__WORKING_ITEMS__.update(item_info)
63
63
 
{cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb/schedulers/scheduler_with_redis.py RENAMED
@@ -69,7 +69,7 @@ class RedisScheduler(Scheduler):
69
69
 
70
70
  seeds, item_info = list(), dict()
71
71
  for member, priority in members:
72
- seed = Seed(member, priority=priority)
72
+ seed = Seed(member, priority=int(priority % 1000))
73
73
  item_info[seed.to_string] = seed.params.priority
74
74
  seeds.append(seed)
75
75
 
@@ -97,12 +97,12 @@ class RedisScheduler(Scheduler):
97
97
  """
98
98
  刷新doing种子过期时间，防止reset重新消费
99
99
  """
100
- time.sleep(20)
101
100
  if item_info := self.get_working_items():
102
101
  refresh_time = int(time.time())
103
102
  seed_info = {k: -refresh_time - v / 1000 for k, v in item_info.items()}
104
103
  self.db.zadd(self.todo_key, seed_info, xx=True)
105
- self.set_working_items(seed_info)
104
+ # self.set_working_items(seed_info)
105
+ time.sleep(20)
106
106
 
107
107
  @check_pause
108
108
  def delete(self):
{cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/cobweb_launcher.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.17
3
+ Version: 3.1.19
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-3.1.17 → cobweb-launcher-3.1.19}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="3.1.17",
8
+ version="3.1.19",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",