cobweb-launcher 3.1.21__tar.gz → 3.1.22__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {cobweb-launcher-3.1.21/cobweb_launcher.egg-info → cobweb-launcher-3.1.22}/PKG-INFO +1 -1
  2. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/schedulers/scheduler_with_redis.py +4 -9
  3. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  4. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/setup.py +1 -1
  5. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/LICENSE +0 -0
  6. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/README.md +0 -0
  7. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/__init__.py +0 -0
  8. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/base/__init__.py +0 -0
  9. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/base/common_queue.py +0 -0
  10. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/base/item.py +0 -0
  11. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/base/logger.py +0 -0
  12. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/base/request.py +0 -0
  13. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/base/response.py +0 -0
  14. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/base/seed.py +0 -0
  15. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/constant.py +0 -0
  16. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/crawlers/__init__.py +0 -0
  17. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/crawlers/crawler.py +0 -0
  18. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/db/__init__.py +0 -0
  19. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/db/api_db.py +0 -0
  20. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/db/redis_db.py +0 -0
  21. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/exceptions/__init__.py +0 -0
  22. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/exceptions/oss_db_exception.py +0 -0
  23. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/launchers/__init__.py +0 -0
  24. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/launchers/distributor.py +0 -0
  25. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/launchers/launcher.py +0 -0
  26. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/launchers/uploader.py +0 -0
  27. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/pipelines/__init__.py +0 -0
  28. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/pipelines/pipeline.py +0 -0
  29. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/pipelines/pipeline_csv.py +0 -0
  30. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/pipelines/pipeline_loghub.py +0 -0
  31. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/schedulers/__init__.py +0 -0
  32. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/schedulers/scheduler.py +0 -0
  33. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/setting.py +0 -0
  34. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/utils/__init__.py +0 -0
  35. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/utils/bloom.py +0 -0
  36. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/utils/decorators.py +0 -0
  37. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/utils/dotting.py +0 -0
  38. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/utils/oss.py +0 -0
  39. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/utils/tools.py +0 -0
  40. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  41. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  42. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb_launcher.egg-info/requires.txt +0 -0
  43. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb_launcher.egg-info/top_level.txt +0 -0
  44. {cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/setup.cfg +0 -0
{cobweb-launcher-3.1.21/cobweb_launcher.egg-info → cobweb-launcher-3.1.22}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.21
3
+ Version: 3.1.22
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/cobweb/schedulers/scheduler_with_redis.py RENAMED
@@ -40,7 +40,8 @@ class RedisScheduler(Scheduler):
40
40
  while not self.stop.is_set():
41
41
  if self.db.lock(self.reset_lock_key, t=60):
42
42
 
43
- _min = -int(time.time()) + self.seed_reset_seconds
43
+ # _min = -int(time.time()) + self.seed_reset_seconds
44
+ _min = -int(time.time()) + 3600
44
45
  self.db.members(self.todo_key, 0, _min=_min, _max="(0")
45
46
  self.db.delete(self.reset_lock_key)
46
47
 
@@ -59,12 +60,6 @@ class RedisScheduler(Scheduler):
59
60
  time.sleep(self.todo_queue_full_wait_seconds)
60
61
  return
61
62
 
62
- count = self.get_working_items_count()
63
- if count > 500:
64
- time.sleep(self.todo_queue_full_wait_seconds)
65
- logger.info(f"任务挤压, 内存任务数：{count}")
66
- return
67
-
68
63
  members = self.db.members(
69
64
  self.todo_key, int(time.time()),
70
65
  count=self.todo_queue_size,
@@ -79,9 +74,9 @@ class RedisScheduler(Scheduler):
79
74
  item_info[seed.to_string] = seed.params.priority
80
75
  seeds.append(seed)
81
76
 
82
- self.set_working_items(item_info)
77
+ # self.set_working_items(item_info)
83
78
  self.todo.push(seeds)
84
- time.sleep(1)
79
+ # time.sleep(1)
85
80
 
86
81
  @check_pause
87
82
  def insert(self):
{cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.21
3
+ Version: 3.1.22
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-3.1.21 → cobweb-launcher-3.1.22}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="3.1.21",
8
+ version="3.1.22",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",