cobweb-launcher 0.1.18__tar.gz → 0.1.19__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (47) hide show
  1. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/PKG-INFO +1 -1
  2. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/constant.py +1 -0
  3. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/single/models.py +4 -4
  4. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/task.py +1 -0
  5. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb_launcher.egg-info/PKG-INFO +1 -1
  6. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/setup.py +1 -1
  7. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/LICENSE +0 -0
  8. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/README.md +0 -0
  9. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/__init__.py +0 -0
  10. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/bbb.py +0 -0
  11. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/__init__.py +0 -0
  12. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/oss_db.py +0 -0
  13. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/redis_db.py +0 -0
  14. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/scheduler/__init__.py +0 -0
  15. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/scheduler/default.py +0 -0
  16. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/scheduler/textfile.py +0 -0
  17. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/storer/__init__.py +0 -0
  18. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/storer/console.py +0 -0
  19. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/storer/loghub.py +0 -0
  20. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/storer/redis.py +0 -0
  21. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/db/storer/textfile.py +0 -0
  22. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/decorators.py +0 -0
  23. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/distributed/__init__.py +0 -0
  24. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/distributed/launcher.py +0 -0
  25. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/distributed/models.py +0 -0
  26. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/__init__.py +0 -0
  27. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/dev/__init__.py +0 -0
  28. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/dev/launcher.py +0 -0
  29. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/dev/models.py +0 -0
  30. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/distributed/__init__.py +0 -0
  31. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/distributed/launcher.py +0 -0
  32. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/distributed/models.py +0 -0
  33. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/single/__init__.py +0 -0
  34. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/single/launcher.py +0 -0
  35. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/interface.py +0 -0
  36. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/log.py +0 -0
  37. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/setting.py +0 -0
  38. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/single/__init__.py +0 -0
  39. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/single/launcher.py +0 -0
  40. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/single/models.py +0 -0
  41. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/single/nest.py +0 -0
  42. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/utils.py +0 -0
  43. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  44. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  45. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb_launcher.egg-info/requires.txt +0 -0
  46. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb_launcher.egg-info/top_level.txt +0 -0
  47. {cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/setup.cfg +0 -0
{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 0.1.18
3
+ Version: 0.1.19
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/constant.py RENAMED
@@ -25,3 +25,4 @@ class Setting:
25
25
  SCHEDULER_WAIT_TIME = None
26
26
  SCHEDULER_BLOCK_TIME = None
27
27
  SPIDER_WAIT_TIME = None
28
+ SPIDER_SLEEP_TIME = None
{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/equip/single/models.py RENAMED
@@ -33,11 +33,11 @@ class Scheduler:
33
33
  while not stop.is_set():
34
34
 
35
35
  if not ready_seed_length():
36
- time.sleep(5)
36
+ time.sleep(Setting.SCHEDULER_WAIT_TIME)
37
37
  continue
38
38
 
39
39
  if self.queue.length >= self.length:
40
- time.sleep(3)
40
+ time.sleep(Setting.SCHEDULER_BLOCK_TIME)
41
41
  continue
42
42
 
43
43
  seeds = get_seed(self.length)
@@ -59,7 +59,7 @@ class Spider:
59
59
  seed = self.queue.pop()
60
60
 
61
61
  if not seed:
62
- time.sleep(3)
62
+ time.sleep(Setting.SPIDER_WAIT_TIME)
63
63
  continue
64
64
 
65
65
  elif seed._retry >= self.max_retries:
@@ -122,7 +122,7 @@ class Spider:
122
122
  log.info(f"{str(seed)} -> {str(e)}")
123
123
  finally:
124
124
  self.spider_in_progress.pop()
125
- time.sleep(Setting.SPIDER_RUN_TIME)
125
+ time.sleep(Setting.SPIDER_SLEEP_TIME)
126
126
  log.info(f"close thread: spider")
127
127
 
128
128
 
{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb/task.py RENAMED
@@ -11,6 +11,7 @@ def init_task_env():
11
11
  Setting.SCHEDULER_WAIT_TIME = float(os.getenv("SCHEDULER_WAIT_TIME", 5))
12
12
  Setting.SCHEDULER_BLOCK_TIME = float(os.getenv("SCHEDULER_BLOCK_TIME", 3))
13
13
  Setting.SPIDER_WAIT_TIME = float(os.getenv("SPIDER_WAIT_TIME", 3))
14
+ Setting.SPIDER_SLEEP_TIME = float(os.getenv("SPIDER_SLEEP_TIME", 0.5))
14
15
 
15
16
 
16
17
  class Task:
{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/cobweb_launcher.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 0.1.18
3
+ Version: 0.1.19
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.19}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="0.1.18",
8
+ version="0.1.19",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",