cobweb-launcher 1.0.9__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (37) hide show
  1. {cobweb-launcher-1.0.9/cobweb_launcher.egg-info → cobweb-launcher-1.1.0}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/crawlers/base_crawler.py +1 -0
  3. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/db/redis_db.py +1 -0
  4. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/launchers/launcher_pro.py +2 -5
  5. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/setting.py +1 -1
  6. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  7. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/setup.py +1 -1
  8. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/LICENSE +0 -0
  9. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/README.md +0 -0
  10. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/__init__.py +0 -0
  11. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/base/__init__.py +0 -0
  12. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/base/common_queue.py +0 -0
  13. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/base/decorators.py +0 -0
  14. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/base/item.py +0 -0
  15. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/base/log.py +0 -0
  16. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/base/request.py +0 -0
  17. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/base/response.py +0 -0
  18. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/base/seed.py +0 -0
  19. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/constant.py +0 -0
  20. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/crawlers/__init__.py +0 -0
  21. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/crawlers/file_crawler.py +0 -0
  22. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/db/__init__.py +0 -0
  23. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/exceptions/__init__.py +0 -0
  24. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/exceptions/oss_db_exception.py +0 -0
  25. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/launchers/__init__.py +0 -0
  26. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/launchers/launcher.py +0 -0
  27. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/pipelines/__init__.py +0 -0
  28. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/pipelines/base_pipeline.py +0 -0
  29. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/pipelines/loghub_pipeline.py +0 -0
  30. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/utils/__init__.py +0 -0
  31. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/utils/oss.py +0 -0
  32. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/utils/tools.py +0 -0
  33. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  34. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  35. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb_launcher.egg-info/requires.txt +0 -0
  36. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb_launcher.egg-info/top_level.txt +0 -0
  37. {cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/setup.cfg +0 -0
{cobweb-launcher-1.0.9/cobweb_launcher.egg-info → cobweb-launcher-1.1.0}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.0.9
3
+ Version: 1.1.0
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/crawlers/base_crawler.py RENAMED
@@ -54,6 +54,7 @@ class Crawler(threading.Thread):
54
54
  continue
55
55
 
56
56
  elif seed.params.retry >= self.max_retries:
57
+ seed.params.identifier = DealModel.fail
57
58
  self.launcher_queue['done'].push(seed)
58
59
  continue
59
60
 
{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/db/redis_db.py RENAMED
@@ -126,3 +126,4 @@ class RedisDB:
126
126
  self.execute_lua(lua_script, keys, *args)
127
127
 
128
128
 
129
+
{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/launchers/launcher_pro.py RENAMED
@@ -16,7 +16,6 @@ class LauncherPro(Launcher):
16
16
  self._fail = "{%s:%s}:fail" % (project, task)
17
17
  self._heartbeat = "heartbeat:%s_%s" % (project, task)
18
18
  self._reset_lock = "lock:reset:%s_%s" % (project, task)
19
- self._heartbeat_lock = "lock:heartbeat:%s_%s" % (project, task)
20
19
  self._db = RedisDB()
21
20
 
22
21
  self._heartbeat_start_event = threading.Event()
@@ -29,10 +28,8 @@ class LauncherPro(Launcher):
29
28
  def _execute_heartbeat(self):
30
29
  while not self._stop.is_set():
31
30
  if self._heartbeat_start_event.is_set():
32
- if self._db.lock(self._heartbeat_lock, t=1):
33
- self._db.setex(self._heartbeat, 3)
34
- time.sleep(1)
35
- time.sleep(0.5)
31
+ self._db.setex(self._heartbeat, 3)
32
+ time.sleep(2)
36
33
 
37
34
  def _reset(self):
38
35
  """
{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/cobweb/setting.py RENAMED
@@ -23,8 +23,8 @@ OSS_BUCKET = os.getenv("OSS_BUCKET")
23
23
  OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
24
24
  OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
25
25
  OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
26
+ OSS_CHUNK_SIZE = 10 * 1024 ** 2
26
27
  OSS_MIN_UPLOAD_SIZE = 1024
27
- OSS_CHUNK_SIZE = 1024 ** 2
28
28
 
29
29
  # 采集器选择
30
30
  CRAWLER = "cobweb.crawlers.CrawlerAir"
{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.0.9
3
+ Version: 1.1.0
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.0.9",
8
+ version="1.1.0",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",
{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.0}/LICENSE RENAMED
File without changes