cobweb-launcher 3.1.3__tar.gz → 3.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/PKG-INFO +1 -1
  2. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/launchers/uploader.py +5 -5
  3. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb_launcher.egg-info/PKG-INFO +1 -1
  4. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/setup.py +1 -1
  5. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/LICENSE +0 -0
  6. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/README.md +0 -0
  7. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/__init__.py +0 -0
  8. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/base/__init__.py +0 -0
  9. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/base/common_queue.py +0 -0
  10. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/base/item.py +0 -0
  11. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/base/log.py +0 -0
  12. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/base/request.py +0 -0
  13. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/base/response.py +0 -0
  14. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/base/seed.py +0 -0
  15. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/constant.py +0 -0
  16. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/crawlers/__init__.py +0 -0
  17. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/crawlers/crawler.py +0 -0
  18. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/db/__init__.py +0 -0
  19. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/db/api_db.py +0 -0
  20. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/db/redis_db.py +0 -0
  21. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/exceptions/__init__.py +0 -0
  22. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/exceptions/oss_db_exception.py +0 -0
  23. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/launchers/__init__.py +0 -0
  24. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/launchers/distributor.py +0 -0
  25. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/launchers/launcher.py +0 -0
  26. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/pipelines/__init__.py +0 -0
  27. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/pipelines/pipeline.py +0 -0
  28. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/pipelines/pipeline_console.py +0 -0
  29. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/pipelines/pipeline_loghub.py +0 -0
  30. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/schedulers/__init__.py +0 -0
  31. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/schedulers/launcher_air.py +0 -0
  32. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/schedulers/launcher_api.py +0 -0
  33. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/schedulers/scheduler.py +0 -0
  34. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/schedulers/scheduler_with_redis.py +0 -0
  35. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/setting.py +0 -0
  36. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/utils/__init__.py +0 -0
  37. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/utils/bloom.py +0 -0
  38. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/utils/decorators.py +0 -0
  39. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/utils/dotting.py +0 -0
  40. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/utils/oss.py +0 -0
  41. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/utils/tools.py +0 -0
  42. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  43. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  44. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb_launcher.egg-info/requires.txt +0 -0
  45. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb_launcher.egg-info/top_level.txt +0 -0
  46. {cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/setup.cfg +0 -0
{cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.3
3
+ Version: 3.1.5
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb/launchers/uploader.py RENAMED
@@ -27,12 +27,12 @@ class Uploader(threading.Thread):
27
27
  self.upload_size = setting.UPLOAD_QUEUE_MAX_SIZE
28
28
  self.wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
29
29
 
30
- self.Pipeline = SpiderPipeline
30
+ self.pipeline = SpiderPipeline()
31
31
 
32
32
  logger.debug(f"Uploader instance attrs: {self.__dict__}")
33
33
 
34
34
  @check_pause
35
- def upload(self):
35
+ def upload_data(self):
36
36
  if not self.upload.length:
37
37
  time.sleep(self.wait_seconds)
38
38
  return
@@ -45,11 +45,11 @@ class Uploader(threading.Thread):
45
45
  if not item:
46
46
  break
47
47
  seeds.append(item.seed)
48
- data = self.Pipeline.build(item)
48
+ data = self.pipeline.build(item)
49
49
  data_info.setdefault(item.table, []).append(data)
50
50
  for table, datas in data_info.items():
51
51
  try:
52
- self.Pipeline.upload(table, datas)
52
+ self.pipeline.upload(table, datas)
53
53
  except Exception as e:
54
54
  logger.info(e)
55
55
  except Exception as e:
@@ -60,6 +60,6 @@ class Uploader(threading.Thread):
60
60
  logger.info("upload pipeline close!")
61
61
 
62
62
  def run(self):
63
- self.register(self.upload, tag="Uploader")
63
+ self.register(self.upload_data, tag="Uploader")
64
64
 
65
65
 
{cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/cobweb_launcher.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.3
3
+ Version: 3.1.5
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="3.1.3",
8
+ version="3.1.5",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",
{cobweb-launcher-3.1.3 → cobweb-launcher-3.1.5}/LICENSE RENAMED
File without changes