cobweb-launcher 3.1.15__tar.gz → 3.1.16__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/PKG-INFO +1 -1
  2. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/schedulers/scheduler.py +6 -3
  3. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/schedulers/scheduler_with_redis.py +6 -4
  4. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb_launcher.egg-info/PKG-INFO +1 -1
  5. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/setup.py +1 -1
  6. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/LICENSE +0 -0
  7. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/README.md +0 -0
  8. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/__init__.py +0 -0
  9. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/base/__init__.py +0 -0
  10. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/base/common_queue.py +0 -0
  11. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/base/item.py +0 -0
  12. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/base/logger.py +0 -0
  13. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/base/request.py +0 -0
  14. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/base/response.py +0 -0
  15. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/base/seed.py +0 -0
  16. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/constant.py +0 -0
  17. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/crawlers/__init__.py +0 -0
  18. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/crawlers/crawler.py +0 -0
  19. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/db/__init__.py +0 -0
  20. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/db/api_db.py +0 -0
  21. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/db/redis_db.py +0 -0
  22. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/exceptions/__init__.py +0 -0
  23. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/exceptions/oss_db_exception.py +0 -0
  24. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/launchers/__init__.py +0 -0
  25. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/launchers/distributor.py +0 -0
  26. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/launchers/launcher.py +0 -0
  27. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/launchers/uploader.py +0 -0
  28. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/pipelines/__init__.py +0 -0
  29. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/pipelines/pipeline.py +0 -0
  30. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/pipelines/pipeline_csv.py +0 -0
  31. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/pipelines/pipeline_loghub.py +0 -0
  32. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/schedulers/__init__.py +0 -0
  33. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/setting.py +0 -0
  34. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/utils/__init__.py +0 -0
  35. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/utils/bloom.py +0 -0
  36. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/utils/decorators.py +0 -0
  37. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/utils/dotting.py +0 -0
  38. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/utils/oss.py +0 -0
  39. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/utils/tools.py +0 -0
  40. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  41. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  42. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb_launcher.egg-info/requires.txt +0 -0
  43. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb_launcher.egg-info/top_level.txt +0 -0
  44. {cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/setup.cfg +0 -0
{cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.15
3
+ Version: 3.1.16
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/schedulers/scheduler.py RENAMED
@@ -68,9 +68,12 @@ class Scheduler(ABC, threading.Thread):
68
68
  def remove_working_items(self, items: list[str] = None):
69
69
  if not items:
70
70
  return
71
- with self.lock:
72
- for item in items:
73
- self.__WORKING_ITEMS__.pop(item, None)
71
+ for _ in range(5):
72
+ with self.lock:
73
+ for item in items:
74
+ self.__WORKING_ITEMS__.pop(item, None)
75
+ return True
76
+ time.sleep(1)
74
77
 
75
78
  def get_working_items_count(self) -> int:
76
79
  with self.lock:
{cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb/schedulers/scheduler_with_redis.py RENAMED
@@ -109,11 +109,13 @@ class RedisScheduler(Scheduler):
109
109
  """
110
110
  删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
111
111
  """
112
- seeds = [seed.to_string for seed in self.done.iter_items(limit=self.done_queue_max_size)]
112
+ seeds = [seed for seed in self.done.iter_items(limit=self.done_queue_max_size)]
113
+ items = [seed.to_string for seed in seeds]
113
114
 
114
- if seeds:
115
- self.db.zrem(self.todo_key, *seeds)
116
- self.remove_working_items(seeds)
115
+ if self.remove_working_items(items):
116
+ self.db.zrem(self.todo_key, *items)
117
+ else:
118
+ self.done.push(seeds)
117
119
 
118
120
  if self.done.length < self.done_queue_max_size:
119
121
  time.sleep(self.done_queue_wait_seconds)
{cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/cobweb_launcher.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.15
3
+ Version: 3.1.16
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-3.1.15 → cobweb-launcher-3.1.16}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="3.1.15",
8
+ version="3.1.16",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",