cobweb-launcher 1.2.65__tar.gz → 1.2.67__tar.gz

{cobweb-launcher-1.2.65/cobweb_launcher.egg-info → cobweb-launcher-1.2.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.65
+Version: 1.2.67
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.65 → cobweb-launcher-1.2.67}/cobweb/crawlers/crawler.py

@@ -36,6 +36,7 @@ class Crawler(threading.Thread):
             delete_seed: Callable,
             upload_data: Callable,
             custom_func: Union[Mapping[str, Callable]],
+            record_failed: bool,
             thread_num: int,
             max_retries: int,
             time_sleep: int,
@@ -50,6 +51,7 @@ class Crawler(threading.Thread):
         self._add_seed = add_seed
         self._delete_seed = delete_seed
         self._upload_data = upload_data
+        self._record_failed = record_failed
 
         for func_name, _callable in custom_func.items():
             if isinstance(_callable, Callable):
@@ -105,7 +107,10 @@ class Crawler(threading.Thread):
 
             elif seed.params.retry > self.max_retries:
                 seed.params.seed_status = DealModel.fail
-                self._delete_seed(seed)
+                if self._record_failed:
+                    self.parse(Response(seed, "max_retries", fialed=True))
+                else:
+                    self._delete_seed(seed)
                 continue
 
             seed_detail_log_info = LogTemplate.log_info(seed.to_dict)

{cobweb-launcher-1.2.65 → cobweb-launcher-1.2.67}/cobweb/launchers/launcher.py

@@ -98,6 +98,7 @@ class Launcher(threading.Thread):
         self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
 
         self._spider_max_retries = setting.SPIDER_MAX_RETRIES
+        self._record_failed = setting.RECORD_FAILED_SPIDER
         self._spider_thread_num = setting.SPIDER_THREAD_NUM
         self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
         self._spider_max_count = setting.SPIDER_MAX_COUNT
@@ -215,6 +216,7 @@ class Launcher(threading.Thread):
             delete_seed=self._delete_seed,
             upload_data=self._upload_data,
             custom_func=self.__CUSTOM_FUNC__,
+            record_failed=self._record_failed,
             thread_num = self._spider_thread_num,
             max_retries = self._spider_max_retries,
             time_sleep=self._spider_time_sleep

{cobweb-launcher-1.2.65 → cobweb-launcher-1.2.67}/cobweb/setting.py

@@ -58,6 +58,7 @@ DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
 SPIDER_TIME_SLEEP = 10
+RECORD_FAILED_SPIDER = False
 
 SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
 TIME_WINDOW = 60   # 频控固定时间窗口（秒）

{cobweb-launcher-1.2.65 → cobweb-launcher-1.2.67/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.65
+Version: 1.2.67
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.65 → cobweb-launcher-1.2.67}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.65",
+    version="1.2.67",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",