cobweb-launcher 1.1.2__tar.gz → 1.1.4__tar.gz

{cobweb-launcher-1.1.2/cobweb_launcher.egg-info → cobweb-launcher-1.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.2
+Version: 1.1.4
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.2 → cobweb-launcher-1.1.4}/cobweb/__init__.py

@@ -1 +1,2 @@
 from .launchers import Launcher, LauncherPro
+from .constant import Crawler

{cobweb-launcher-1.1.2 → cobweb-launcher-1.1.4}/cobweb/constant.py

@@ -1,4 +1,10 @@
 
+class Crawler:
+
+    default = "cobweb.crawlers.Crawler"
+    file_air = "cobweb.crawlers.FileCrawlerAir"
+    file_pro = "cobweb.crawlers.FileCrawlerPro"
+
 
 class LauncherModel:
     task = "launcher model: task"

cobweb-launcher-1.1.4/cobweb/crawlers/__init__.py

@@ -0,0 +1,2 @@
+from .base_crawler import Crawler
+from .file_crawler import FileCrawlerAir, FileCrawlerPro

{cobweb-launcher-1.1.2 → cobweb-launcher-1.1.4}/cobweb/crawlers/file_crawler.py

@@ -7,18 +7,19 @@ from cobweb.base import Seed, BaseItem, Request, Response
 from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
 
 
-class CrawlerAir(Crawler):
-    
-    oss_util = OssUtil()
+oss_util = OssUtil()
+
+
+class FileCrawlerAir(Crawler):
 
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
         seed_dict = item.seed.to_dict
-        bucket_name = CrawlerAir.oss_util.bucket
+        bucket_name = oss_util.bucket
         try:
             key = item.seed.oss_path
-            if CrawlerAir.oss_util.exists(key):
-                content_length = CrawlerAir.oss_util.head(key).content_length
+            if oss_util.exists(key):
+                content_length = oss_util.head(key).content_length
                 yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
 
             end = seed_dict.get("end", "")
@@ -29,8 +30,8 @@ class CrawlerAir(Crawler):
 
             if not item.seed.params.identifier:
                 content = b""
-                chunk_size = CrawlerAir.oss_util.chunk_size
-                min_upload_size = CrawlerAir.oss_util.min_upload_size
+                chunk_size = oss_util.chunk_size
+                min_upload_size = oss_util.min_upload_size
                 position = seed_dict.get("position", 1)
 
                 response = item.download()
@@ -52,21 +53,21 @@ class CrawlerAir(Crawler):
                     """小文件直接下载"""
                     for part_data in response.iter_content(chunk_size):
                         content += part_data
-                    CrawlerAir.oss_util.put(key, content)
+                    oss_util.put(key, content)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
                     response.close()
                 else:
                     """中大文件同步分片下载"""
                     upload_content_length = 0
                     if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = CrawlerAir.oss_util.init_part(key).upload_id
+                        seed_dict["upload_id"] = oss_util.init_part(key).upload_id
                     upload_id = seed_dict["upload_id"]
                     for part_data in response.iter_content(chunk_size):
                         content += part_data
                         if len(content) >= chunk_size:
                             upload_data = content[:chunk_size]
                             content = content[chunk_size:]
-                            CrawlerAir.oss_util.put_part(key, upload_id, position, upload_data)
+                            oss_util.put_part(key, upload_id, position, upload_data)
                             upload_content_length += len(upload_data)
                             position += 1
                             seed_dict['position'] = position
@@ -74,14 +75,14 @@ class CrawlerAir(Crawler):
 
                     response.close()
                     if content:
-                        CrawlerAir.oss_util.put_part(key, upload_id, position, content)
+                        oss_util.put_part(key, upload_id, position, content)
                         content_length += len(content)
-                    CrawlerAir.oss_util.merge(key, upload_id)
+                    oss_util.merge(key, upload_id)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
 
             elif item.seed.params.identifier == "merge":
-                CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
-                content_length = CrawlerAir.oss_util.head(key).content_length
+                oss_util.merge(key, seed_dict["upload_id"])
+                content_length = oss_util.head(key).content_length
                 yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
         except OssDBPutPartError:
             yield Seed(seed_dict)
@@ -89,18 +90,16 @@ class CrawlerAir(Crawler):
             yield Seed(seed_dict, identifier="merge")
 
 
-class CrawlerPro(Crawler):
-
-    oss_util = OssUtil()
+class FileCrawlerPro(Crawler):
 
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
         seed_dict = item.seed.to_dict
-        bucket_name = CrawlerAir.oss_util.bucket
+        bucket_name = oss_util.bucket
         try:
             key = item.seed.oss_path
-            if CrawlerAir.oss_util.exists(key):
-                content_length = CrawlerAir.oss_util.head(key).content_length
+            if oss_util.exists(key):
+                content_length = oss_util.head(key).content_length
                 yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
 
             end = seed_dict.get("end", "")
@@ -111,8 +110,8 @@ class CrawlerPro(Crawler):
 
             if not item.seed.params.identifier:
                 content = b""
-                chunk_size = CrawlerAir.oss_util.chunk_size
-                min_upload_size = CrawlerAir.oss_util.min_upload_size
+                chunk_size = oss_util.chunk_size
+                min_upload_size = oss_util.min_upload_size
                 position = seed_dict.get("position", 1)
 
                 response = item.download()
@@ -134,21 +133,21 @@ class CrawlerPro(Crawler):
                     """小文件直接下载"""
                     for part_data in response.iter_content(chunk_size):
                         content += part_data
-                    CrawlerAir.oss_util.put(key, content)
+                    oss_util.put(key, content)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
                     response.close()
                 else:
                     """中大文件同步分片下载"""
                     upload_content_length = 0
                     if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = CrawlerAir.oss_util.init_part(key).upload_id
+                        seed_dict["upload_id"] = oss_util.init_part(key).upload_id
                     upload_id = seed_dict["upload_id"]
                     for part_data in response.iter_content(chunk_size):
                         content += part_data
                         if len(content) >= chunk_size:
                             upload_data = content[:chunk_size]
                             content = content[chunk_size:]
-                            CrawlerAir.oss_util.put_part(key, upload_id, position, upload_data)
+                            oss_util.put_part(key, upload_id, position, upload_data)
                             upload_content_length += len(upload_data)
                             position += 1
                             seed_dict['position'] = position
@@ -156,14 +155,14 @@ class CrawlerPro(Crawler):
 
                     response.close()
                     if content:
-                        CrawlerAir.oss_util.put_part(key, upload_id, position, content)
+                        oss_util.put_part(key, upload_id, position, content)
                         content_length += len(content)
-                    CrawlerAir.oss_util.merge(key, upload_id)
+                    oss_util.merge(key, upload_id)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
 
             elif item.seed.params.identifier == "merge":
-                CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
-                content_length = CrawlerAir.oss_util.head(key).content_length
+                oss_util.merge(key, seed_dict["upload_id"])
+                content_length = oss_util.head(key).content_length
                 yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
 
         except OssDBPutPartError:

{cobweb-launcher-1.1.2 → cobweb-launcher-1.1.4}/cobweb/launchers/launcher_pro.py

@@ -35,25 +35,24 @@ class LauncherPro(Launcher):
         """
         检查过期种子，重新添加到redis缓存中
         """
-        first = True
+        init = True
         while not self._pause.is_set():
-            reset_wait_seconds = 15
+            reset_wait_seconds = 30
+            start_reset_time = int(time.time())
             if self._db.lock(self._reset_lock, t=120):
                 if not self.heartbeat:
                     self._heartbeat_start_event.set()
 
                 _min = -int(time.time()) + self._seed_reset_seconds \
-                    if self.heartbeat or not first else "-inf"
+                    if self.heartbeat or not init else "-inf"
 
-                self._db.members(
-                    self._todo, 0,
-                    _min=_min, _max="(0"
-                )
+                self._db.members(self._todo, 0, _min=_min, _max="(0")
                 self._db.delete(self._reset_lock)
-                reset_wait_seconds = 60
 
+                ttl = 120 - int(time.time()) + start_reset_time
+                reset_wait_seconds = max(ttl, 1)
             time.sleep(reset_wait_seconds)
-            first = False
+            init = False
 
     def _scheduler(self):
         """

{cobweb-launcher-1.1.2 → cobweb-launcher-1.1.4}/cobweb/setting.py

@@ -27,7 +27,7 @@ OSS_CHUNK_SIZE = 10 * 1024 ** 2
 OSS_MIN_UPLOAD_SIZE = 1024
 
 # 采集器选择
-CRAWLER = "cobweb.crawlers.CrawlerAir"
+CRAWLER = "cobweb.crawlers.Crawler"
 
 # 数据上传链路
 PIPELINE = "cobweb.pipelines.loghub_pipeline.LoghubPipeline"
@@ -39,7 +39,7 @@ TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
 DONE_QUEUE_WAIT_SECONDS = 15   # done队列等待时间
 UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
-SEED_RESET_SECONDS = 600   # 种子重制时间
+SEED_RESET_SECONDS = 300   # 种子重制时间
 
 
 # Launcher 队列长度

{cobweb-launcher-1.1.2 → cobweb-launcher-1.1.4/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.2
+Version: 1.1.4
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.2 → cobweb-launcher-1.1.4}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.1.2",
+    version="1.1.4",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",

cobweb-launcher-1.1.2/cobweb/crawlers/__init__.py

@@ -1,2 +0,0 @@
-from .base_crawler import Crawler
-from .file_crawler import CrawlerAir