cobweb-launcher 1.2.64__tar.gz → 1.2.66__tar.gz

{cobweb-launcher-1.2.64/cobweb_launcher.egg-info → cobweb-launcher-1.2.66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.64
+Version: 1.2.66
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.64 → cobweb-launcher-1.2.66}/cobweb/crawlers/crawler.py

@@ -36,6 +36,7 @@ class Crawler(threading.Thread):
             delete_seed: Callable,
             upload_data: Callable,
             custom_func: Union[Mapping[str, Callable]],
+            record_failed: bool,
             thread_num: int,
             max_retries: int,
             time_sleep: int,
@@ -50,6 +51,7 @@ class Crawler(threading.Thread):
         self._add_seed = add_seed
         self._delete_seed = delete_seed
         self._upload_data = upload_data
+        self._record_failed = record_failed
 
         for func_name, _callable in custom_func.items():
             if isinstance(_callable, Callable):
@@ -105,7 +107,10 @@ class Crawler(threading.Thread):
 
             elif seed.params.retry > self.max_retries:
                 seed.params.seed_status = DealModel.fail
-                self._delete_seed(seed)
+                if self._record_failed:
+                    self.parse(Response(seed, "failed"))
+                else:
+                    self._delete_seed(seed)
                 continue
 
             seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
@@ -149,6 +154,7 @@ class Crawler(threading.Thread):
                                         data_size=int(download_item.response.headers.get("content-length", 0)),
                                         cost_time=end_time - start_time, status = 200,
                                         url=download_item.response.url,
+                                        seed=download_item.seed.to_string,
                                         proxy_type=seed.params.proxy_type,
                                         proxy=seed.params.proxy,
                                         project=self.project, task=self.task,
@@ -179,6 +185,7 @@ class Crawler(threading.Thread):
                     topic=urlparse(url).netloc,
                     data_size=-1, cost_time=-1,
                     status=status, url=url,
+                    seed=seed.to_string,
                     proxy_type=seed.params.proxy_type,
                     proxy=seed.params.proxy,
                     project=self.project,

{cobweb-launcher-1.2.64 → cobweb-launcher-1.2.66}/cobweb/launchers/launcher.py

@@ -98,6 +98,7 @@ class Launcher(threading.Thread):
         self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
 
         self._spider_max_retries = setting.SPIDER_MAX_RETRIES
+        self._record_failed = setting.RECORD_FAILED_SPIDER
         self._spider_thread_num = setting.SPIDER_THREAD_NUM
         self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
         self._spider_max_count = setting.SPIDER_MAX_COUNT
@@ -215,6 +216,7 @@ class Launcher(threading.Thread):
             delete_seed=self._delete_seed,
             upload_data=self._upload_data,
             custom_func=self.__CUSTOM_FUNC__,
+            record_failed=self._record_failed,
             thread_num = self._spider_thread_num,
             max_retries = self._spider_max_retries,
             time_sleep=self._spider_time_sleep

{cobweb-launcher-1.2.64 → cobweb-launcher-1.2.66}/cobweb/setting.py

@@ -58,6 +58,7 @@ DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
 SPIDER_TIME_SLEEP = 10
+RECORD_FAILED_SPIDER = False
 
 SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
 TIME_WINDOW = 60   # 频控固定时间窗口（秒）

{cobweb-launcher-1.2.64 → cobweb-launcher-1.2.66/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.64
+Version: 1.2.66
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.64 → cobweb-launcher-1.2.66}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.64",
+    version="1.2.66",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",