cobweb-launcher 1.2.64__py3-none-any.whl → 1.2.66__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -36,6 +36,7 @@ class Crawler(threading.Thread):
             delete_seed: Callable,
             upload_data: Callable,
             custom_func: Union[Mapping[str, Callable]],
+            record_failed: bool,
             thread_num: int,
             max_retries: int,
             time_sleep: int,
@@ -50,6 +51,7 @@ class Crawler(threading.Thread):
         self._add_seed = add_seed
         self._delete_seed = delete_seed
         self._upload_data = upload_data
+        self._record_failed = record_failed
 
         for func_name, _callable in custom_func.items():
             if isinstance(_callable, Callable):
@@ -105,7 +107,10 @@ class Crawler(threading.Thread):
 
             elif seed.params.retry > self.max_retries:
                 seed.params.seed_status = DealModel.fail
-                self._delete_seed(seed)
+                if self._record_failed:
+                    self.parse(Response(seed, "failed"))
+                else:
+                    self._delete_seed(seed)
                 continue
 
             seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
@@ -149,6 +154,7 @@ class Crawler(threading.Thread):
                                         data_size=int(download_item.response.headers.get("content-length", 0)),
                                         cost_time=end_time - start_time, status = 200,
                                         url=download_item.response.url,
+                                        seed=download_item.seed.to_string,
                                         proxy_type=seed.params.proxy_type,
                                         proxy=seed.params.proxy,
                                         project=self.project, task=self.task,
@@ -179,6 +185,7 @@ class Crawler(threading.Thread):
                     topic=urlparse(url).netloc,
                     data_size=-1, cost_time=-1,
                     status=status, url=url,
+                    seed=seed.to_string,
                     proxy_type=seed.params.proxy_type,
                     proxy=seed.params.proxy,
                     project=self.project,

cobweb/launchers/launcher.py

@@ -98,6 +98,7 @@ class Launcher(threading.Thread):
         self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
 
         self._spider_max_retries = setting.SPIDER_MAX_RETRIES
+        self._record_failed = setting.RECORD_FAILED_SPIDER
         self._spider_thread_num = setting.SPIDER_THREAD_NUM
         self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
         self._spider_max_count = setting.SPIDER_MAX_COUNT
@@ -215,6 +216,7 @@ class Launcher(threading.Thread):
             delete_seed=self._delete_seed,
             upload_data=self._upload_data,
             custom_func=self.__CUSTOM_FUNC__,
+            record_failed=self._record_failed,
             thread_num = self._spider_thread_num,
             max_retries = self._spider_max_retries,
             time_sleep=self._spider_time_sleep

cobweb/setting.py

@@ -58,6 +58,7 @@ DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
 SPIDER_TIME_SLEEP = 10
+RECORD_FAILED_SPIDER = False
 
 SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
 TIME_WINDOW = 60   # 频控固定时间窗口（秒）

{cobweb_launcher-1.2.64.dist-info → cobweb_launcher-1.2.66.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.64
+Version: 1.2.66
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.64.dist-info → cobweb_launcher-1.2.66.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=LlWZeZLVMeCd9NRCPkFDf_DK0IUua1L2U_zYFPJpn_k,2273
+cobweb/setting.py,sha256=YlNrQSUgly5ah1-7Mx1RWypA5cPcaEIGV4neNJItGZ4,2302
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/basic.py,sha256=Z56SSLB3I2IGHWCCcSy0Qbfzj8Qbg_po3gP32q1jh4k,7741
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
@@ -13,7 +13,7 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=A-F1urjbE5hYNWTCwq3sUV4nrxlK_RGMoCmjBmIwYsI,3158
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=kbpgBllmA2ve3Hp-XvVH89t2q5G39i-m7it5xJ1p1WE,8973
+cobweb/crawlers/crawler.py,sha256=ZSau2W7YMR_XiJmHlkmAJbeH2sbWyma_yGEY2h70yVs,9289
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -22,7 +22,7 @@ cobweb/db/redis_db_new.py,sha256=F09LWVjtC2JFdCaKatZ2bAOLKbsnes85_nZRe2dtSIc,469
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
-cobweb/launchers/launcher.py,sha256=aR1cnQymjQQUJe-W_dRoQTd9t3Qav9QkLDzeWW7i0xE,7921
+cobweb/launchers/launcher.py,sha256=m6NKrVxcqEGXctIAAXXGEsdSFAyK0t49AWxQDCKGJiY,8027
 cobweb/launchers/launcher_air.py,sha256=qdcxq41I9zN5snEpMUUNEQNTtUiZM1Hw_3N9zu4PuAs,3058
 cobweb/launchers/launcher_api.py,sha256=3-A6k3Igvi-xnvP9M_3NkJKUweDJ_pY10ZrHClteD-g,8628
 cobweb/launchers/launcher_pro.py,sha256=xM2z3nCTb6gehuD1Ggyvli2knGAPjGczbjOhKWswVpo,8424
@@ -38,8 +38,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=6x_ugXanh1R-6ZylQzUDQh4OeFZHujhWFCOxbzy53JY,3984
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.64.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.64.dist-info/METADATA,sha256=NikLvEH211SWsw5jTUhaLsXROPJWvc2bkEUq3n4wlaM,6510
-cobweb_launcher-1.2.64.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.64.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.64.dist-info/RECORD,,
+cobweb_launcher-1.2.66.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.66.dist-info/METADATA,sha256=jmTWYE_UNh52CVZ9lV5D6THP36hrkcWAaawHkKvLH_s,6510
+cobweb_launcher-1.2.66.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.66.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.66.dist-info/RECORD,,