cobweb-launcher 1.3.13__tar.gz → 1.3.15__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {cobweb-launcher-1.3.13/cobweb_launcher.egg-info → cobweb-launcher-1.3.15}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/basic.py +2 -0
  3. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/crawlers/crawler.py +1 -1
  4. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/launchers/launcher_api.py +3 -2
  5. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  6. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/setup.py +1 -1
  7. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/LICENSE +0 -0
  8. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/README.md +0 -0
  9. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/__init__.py +0 -0
  10. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/__init__.py +0 -0
  11. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/common_queue.py +0 -0
  12. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/dotting.py +0 -0
  13. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/item.py +0 -0
  14. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/log.py +0 -0
  15. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/request.py +0 -0
  16. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/response.py +0 -0
  17. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/seed.py +0 -0
  18. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/constant.py +0 -0
  19. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/crawlers/__init__.py +0 -0
  20. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/db/__init__.py +0 -0
  21. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/db/api_db.py +0 -0
  22. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/db/redis_db.py +0 -0
  23. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/exceptions/__init__.py +0 -0
  24. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/exceptions/oss_db_exception.py +0 -0
  25. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/launchers/__init__.py +0 -0
  26. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/launchers/launcher.py +0 -0
  27. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/launchers/launcher_air.py +0 -0
  28. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/launchers/launcher_pro.py +0 -0
  29. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/pipelines/__init__.py +0 -0
  30. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/pipelines/pipeline.py +0 -0
  31. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/pipelines/pipeline_console.py +0 -0
  32. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/pipelines/pipeline_loghub.py +0 -0
  33. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/schedulers/__init__.py +0 -0
  34. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/schedulers/scheduler_api.py +0 -0
  35. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/schedulers/scheduler_redis.py +0 -0
  36. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/setting.py +0 -0
  37. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/utils/__init__.py +0 -0
  38. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/utils/bloom.py +0 -0
  39. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/utils/oss.py +0 -0
  40. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/utils/tools.py +0 -0
  41. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  42. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  43. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb_launcher.egg-info/requires.txt +0 -0
  44. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb_launcher.egg-info/top_level.txt +0 -0
  45. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/setup.cfg +0 -0
  46. {cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/test/test.py +0 -0
{cobweb-launcher-1.3.13/cobweb_launcher.egg-info → cobweb-launcher-1.3.15}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.3.13
3
+ Version: 1.3.15
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/base/basic.py RENAMED
@@ -167,6 +167,8 @@ class Request:
167
167
  kwargs.update(**seed.to_dict)
168
168
  elif isinstance(seed, str):
169
169
  kwargs.update(**json.loads(seed))
170
+ elif isinstance(seed, dict):
171
+ kwargs.update(**seed)
170
172
 
171
173
  for k, v in kwargs.items():
172
174
  if k in self.__class__.__REQUEST_ATTRS__:
{cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/crawlers/crawler.py RENAMED
@@ -34,7 +34,7 @@ class Crawler(threading.Thread):
34
34
 
35
35
  @staticmethod
36
36
  def request(seed: Seed) -> Union[Request, BaseItem]:
37
- yield Request(seed.url, seed, timeout=5)
37
+ yield Request(seed, timeout=5)
38
38
 
39
39
  @staticmethod
40
40
  def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
{cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/cobweb/launchers/launcher_api.py RENAMED
@@ -22,8 +22,9 @@ class LauncherApi(Launcher):
22
22
  ]:
23
23
  if q.length < size:
24
24
  for member, priority in self._scheduler.schedule(key, self.scheduling_size):
25
- q.push(Cls(member, priority=priority))
26
- self.add_working_item(key.split(":")[-1], member, priority)
25
+ item = Cls(member, priority=priority)
26
+ q.push(item)
27
+ self.add_working_item(key.split(":")[-1], item.seed, priority)
27
28
  thread_sleep = 0.1
28
29
  time.sleep(thread_sleep)
29
30
 
{cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.3.13
3
+ Version: 1.3.15
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.3.13 → cobweb-launcher-1.3.15}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.3.13",
8
+ version="1.3.15",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",