cobweb-launcher 1.2.66__py3-none-any.whl → 1.2.68__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -1,7 +1,6 @@
 import json
-import os
-import threading
 import time
+import threading
 import traceback
 from inspect import isgenerator
 from typing import Union, Callable, Mapping
@@ -108,7 +107,7 @@ class Crawler(threading.Thread):
             elif seed.params.retry > self.max_retries:
                 seed.params.seed_status = DealModel.fail
                 if self._record_failed:
-                    self.parse(Response(seed, "failed"))
+                    self.parse(Response(seed, "failed", max_retries=True))
                 else:
                     self._delete_seed(seed)
                 continue
@@ -202,22 +201,6 @@ class Crawler(threading.Thread):
                 ))
                 seed.params.retry += 1
                 self._set_seed(seed)
-                # time.sleep(self.time_sleep * seed.params.retry)
-            # except Exception as e:
-            #     logger.info(LogTemplate.download_exception.format(
-            #         detail=seed_detail_log_info,
-            #         retry=seed.params.retry,
-            #         priority=seed.params.priority,
-            #         seed_version=seed.params.seed_version,
-            #         identifier=seed.identifier or "",
-            #         exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
-            #     ))
-            #     seed.params.retry += 1
-            #     # self._todo.push(seed)
-            #     self._set_seed(seed)
-            #     # time.sleep(self.time_sleep * seed.params.retry)
-            finally:
-                time.sleep(0.1)
         logger.info("spider thread close")
 
     def run(self):

{cobweb_launcher-1.2.66.dist-info → cobweb_launcher-1.2.68.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.66
+Version: 1.2.68
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.66.dist-info → cobweb_launcher-1.2.68.dist-info}/RECORD

@@ -13,7 +13,7 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=A-F1urjbE5hYNWTCwq3sUV4nrxlK_RGMoCmjBmIwYsI,3158
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=ZSau2W7YMR_XiJmHlkmAJbeH2sbWyma_yGEY2h70yVs,9289
+cobweb/crawlers/crawler.py,sha256=_V2jJtrf13Wwg9PwOPtOCPRv0aBIq420JHutEz0Ci0E,8496
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -38,8 +38,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=6x_ugXanh1R-6ZylQzUDQh4OeFZHujhWFCOxbzy53JY,3984
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.66.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.66.dist-info/METADATA,sha256=jmTWYE_UNh52CVZ9lV5D6THP36hrkcWAaawHkKvLH_s,6510
-cobweb_launcher-1.2.66.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.66.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.66.dist-info/RECORD,,
+cobweb_launcher-1.2.68.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.68.dist-info/METADATA,sha256=Rr1Pm1Xd3RDv-ldwwocn1S5OMAIZ4E7AfgBv1tWa5Ck,6510
+cobweb_launcher-1.2.68.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.68.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.68.dist-info/RECORD,,