cobweb-launcher 1.1.17__py3-none-any.whl → 1.1.19__py3-none-any.whl

cobweb/crawlers/base_crawler.py

@@ -69,7 +69,7 @@ class Crawler(threading.Thread):
                 continue
 
             elif seed.params.retry >= self.max_retries:
-                seed.params.identifier = DealModel.fail
+                seed.params.seed_status = DealModel.fail
                 self.launcher_queue['done'].push(seed)
                 continue
 
@@ -83,13 +83,11 @@ class Crawler(threading.Thread):
 
                 for request_item in request_iterators:
 
-                    if isinstance(request_item, BaseItem):
-                        self.upload_queue.push(request_item)
-
-                    elif isinstance(request_item, Request):
+                    if isinstance(request_item, Request):
                         download_iterators = self.download(request_item)
                         if not isgenerator(download_iterators):
                             raise TypeError("download function isn't a generator")
+
                         for download_item in download_iterators:
                             if isinstance(download_item, Response):
                                 logger.info(LogTemplate.download_info.format(
@@ -110,6 +108,8 @@ class Crawler(threading.Thread):
                                     self.distribute(parse_item, seed)
                             else:
                                 self.distribute(download_item, seed)
+                    else:
+                        self.distribute(request_item, seed)
             except Exception as e:
                 logger.info(LogTemplate.download_exception.format(
                     detail=seed_detail_log_info,

{cobweb_launcher-1.1.17.dist-info → cobweb_launcher-1.1.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.17
+Version: 1.1.19
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.17.dist-info → cobweb_launcher-1.1.19.dist-info}/RECORD

@@ -10,7 +10,7 @@ cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
 cobweb/crawlers/__init__.py,sha256=Rr3DTjD-abMA1_FYcQJZYNvQvcWMuEVcsIU6duqHrw4,75
-cobweb/crawlers/base_crawler.py,sha256=X7RdgKCcMcYI8NSyGVXdT03uP9uHsEiX137IeBlkg00,5189
+cobweb/crawlers/base_crawler.py,sha256=6_v71I-opx6JhSKFWa7qF__ivzKwRwAgwGI8nwx3YMQ,5154
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
@@ -25,8 +25,8 @@ cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzX
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=qAl05ybL2Jp6KFjHDHVMfmeBHQmDKPtZleHjHAY7LZc,3277
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.17.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.17.dist-info/METADATA,sha256=j-Mf27c0yJOKyZWxGEg3nOvWJE2fQaKZnyB6-MBEMis,1246
-cobweb_launcher-1.1.17.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.17.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.17.dist-info/RECORD,,
+cobweb_launcher-1.1.19.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.19.dist-info/METADATA,sha256=e-o4DJ4zQ8qbHuR3lllCJOGcAdxACG_je1w_o5hcyTs,1246
+cobweb_launcher-1.1.19.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.19.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.19.dist-info/RECORD,,