cobweb-launcher 1.2.62__tar.gz → 1.2.64__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {cobweb-launcher-1.2.62/cobweb_launcher.egg-info → cobweb-launcher-1.2.64}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/launchers/launcher_api.py +7 -4
  3. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/launchers/launcher_pro.py +3 -3
  4. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  5. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/setup.py +1 -1
  6. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/LICENSE +0 -0
  7. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/README.md +0 -0
  8. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/__init__.py +0 -0
  9. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/base/__init__.py +0 -0
  10. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/base/common_queue.py +0 -0
  11. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/base/decorators.py +0 -0
  12. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/base/item.py +0 -0
  13. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/base/log.py +0 -0
  14. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/base/request.py +0 -0
  15. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/base/response.py +0 -0
  16. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/base/seed.py +0 -0
  17. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/constant.py +0 -0
  18. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/crawlers/__init__.py +0 -0
  19. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/crawlers/base_crawler.py +0 -0
  20. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/crawlers/crawler.py +0 -0
  21. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/crawlers/file_crawler.py +0 -0
  22. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/db/__init__.py +0 -0
  23. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/db/api_db.py +0 -0
  24. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/db/redis_db.py +0 -0
  25. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/db/redis_db_new.py +0 -0
  26. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/exceptions/__init__.py +0 -0
  27. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/exceptions/oss_db_exception.py +0 -0
  28. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/launchers/__init__.py +0 -0
  29. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/launchers/launcher.py +0 -0
  30. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/launchers/launcher_air.py +0 -0
  31. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/pipelines/__init__.py +0 -0
  32. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/pipelines/pipeline.py +0 -0
  33. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/pipelines/pipeline_console.py +0 -0
  34. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/pipelines/pipeline_loghub.py +0 -0
  35. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/setting.py +0 -0
  36. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/utils/__init__.py +0 -0
  37. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/utils/bloom.py +0 -0
  38. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/utils/dotting.py +0 -0
  39. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/utils/oss.py +0 -0
  40. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/utils/tools.py +0 -0
  41. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  42. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  43. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb_launcher.egg-info/requires.txt +0 -0
  44. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb_launcher.egg-info/top_level.txt +0 -0
  45. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/setup.cfg +0 -0
  46. {cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/test/test.py +0 -0
{cobweb-launcher-1.2.62/cobweb_launcher.egg-info → cobweb-launcher-1.2.64}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.62
3
+ Version: 1.2.64
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/launchers/launcher_api.py RENAMED
@@ -49,8 +49,11 @@ class LauncherApi(Launcher):
49
49
  if (self._speed_control and self.__LAUNCHER_QUEUE__["todo"].length and
50
50
  not self._db.auto_incr(self._speed_control_key, t=self._time_window, limit=self._spider_max_count)):
51
51
  expire_time = self._db.ttl(self._speed_control_key)
52
- logger.info(f"Too fast! Please wait {expire_time} seconds...")
53
- time.sleep(expire_time / 2)
52
+ if isinstance(expire_time, int) and expire_time <= -1:
53
+ self._db.delete(self._speed_control_key)
54
+ elif isinstance(expire_time, int):
55
+ logger.info(f"Too fast! Please wait {expire_time} seconds...")
56
+ time.sleep(expire_time / 2)
54
57
  return None
55
58
  seed = self.__LAUNCHER_QUEUE__["todo"].pop()
56
59
  return seed
@@ -115,11 +118,11 @@ class LauncherApi(Launcher):
115
118
  break
116
119
  seed, new_seed = seed_tuple
117
120
  new_seeds[new_seed.to_string] = new_seed.params.priority
118
- del_seeds.add(seed.to_string)
121
+ del_seeds.add(seed)
119
122
  if new_seeds:
120
123
  self._db.zadd(self._todo_key, new_seeds, nx=True)
121
124
  if del_seeds:
122
- self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
125
+ self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
123
126
  if status:
124
127
  time.sleep(self._new_queue_wait_seconds)
125
128
 
{cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/cobweb/launchers/launcher_pro.py RENAMED
@@ -47,7 +47,7 @@ class LauncherPro(Launcher):
47
47
  spider_speed = self._db.get(self._speed_control_key)
48
48
  if int(spider_speed or 0) > self._spider_max_count:
49
49
  expire_time = self._db.ttl(self._speed_control_key)
50
- if expire_time <= -1:
50
+ if isinstance(expire_time, int) and expire_time <= -1:
51
51
  self._db.delete(self._speed_control_key)
52
52
  elif isinstance(expire_time, int):
53
53
  logger.info(f"Too fast! Please wait {expire_time} seconds...")
@@ -117,11 +117,11 @@ class LauncherPro(Launcher):
117
117
  break
118
118
  seed, new_seed = seed_tuple
119
119
  new_seeds[new_seed.to_string] = new_seed.params.priority
120
- del_seeds.add(seed.to_string)
120
+ del_seeds.add(seed)
121
121
  if new_seeds:
122
122
  self._db.zadd(self._todo_key, new_seeds, nx=True)
123
123
  if del_seeds:
124
- self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
124
+ self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
125
125
  if status:
126
126
  time.sleep(self._new_queue_wait_seconds)
127
127
 
{cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.62
3
+ Version: 1.2.64
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.62 → cobweb-launcher-1.2.64}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.2.62",
8
+ version="1.2.64",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",