cobweb-launcher 1.1.14__py3-none-any.whl → 1.1.15__py3-none-any.whl

cobweb/crawlers/base_crawler.py

@@ -75,24 +75,23 @@ class Crawler(threading.Thread):
 
             seed_detail_log_info = download_log_info(seed.to_dict)
 
-            request_iterators = self.request(seed)
+            try:
+                request_iterators = self.request(seed)
 
-            if not isgenerator(request_iterators):
-                raise TypeError("request function isn't a generator!")
+                if not isgenerator(request_iterators):
+                    raise TypeError("request function isn't a generator!")
 
-            for request_item in request_iterators:
+                for request_item in request_iterators:
 
-                if isinstance(request_item, BaseItem):
-                    self.upload_queue.push(request_item)
+                    if isinstance(request_item, BaseItem):
+                        self.upload_queue.push(request_item)
 
-                elif isinstance(request_item, Request):
-                    try:
+                    elif isinstance(request_item, Request):
                         download_iterators = self.download(request_item)
                         if not isgenerator(download_iterators):
                             raise TypeError("download function isn't a generator")
                         for download_item in download_iterators:
                             if isinstance(download_item, Response):
-                                response_detail_log_info = download_log_info(download_item.to_dict)
                                 logger.info(LogTemplate.download_info.format(
                                     detail=seed_detail_log_info,
                                     retry=seed.params.retry,
@@ -100,7 +99,7 @@ class Crawler(threading.Thread):
                                     seed_version=seed.params.seed_version,
                                     identifier=seed.identifier or "",
                                     status=download_item.response,
-                                    response=response_detail_log_info
+                                    response=download_log_info(download_item.to_dict)
                                 ))
                                 parse_iterators = self.parse(download_item)
                                 if not isgenerator(parse_iterators):
@@ -111,16 +110,16 @@ class Crawler(threading.Thread):
                                     self.distribute(parse_item, seed)
                             else:
                                 self.distribute(download_item, seed)
-                    except Exception as e:
-                        logger.info(LogTemplate.download_exception.format(
-                            detail=seed_detail_log_info,
-                            retry=seed.params.retry,
-                            priority=seed.params.priority,
-                            seed_version=seed.params.seed_version,
-                            identifier=seed.identifier or "", exception=e
-                        ))
-                        seed.params.retry += 1
-                        self.launcher_queue['todo'].push(seed)
+            except Exception as e:
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info,
+                    retry=seed.params.retry,
+                    priority=seed.params.priority,
+                    seed_version=seed.params.seed_version,
+                    identifier=seed.identifier or "", exception=e
+                ))
+                seed.params.retry += 1
+                self.launcher_queue['todo'].push(seed)
 
     def run(self):
         for index in range(self.spider_thread_num):

{cobweb_launcher-1.1.14.dist-info → cobweb_launcher-1.1.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.14
+Version: 1.1.15
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.14.dist-info → cobweb_launcher-1.1.15.dist-info}/RECORD

@@ -10,7 +10,7 @@ cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=QxlXztWjV8VvcHu-cTzyoDHaDvoX26iyoJKiWk64HnE,2759
 cobweb/crawlers/__init__.py,sha256=Rr3DTjD-abMA1_FYcQJZYNvQvcWMuEVcsIU6duqHrw4,75
-cobweb/crawlers/base_crawler.py,sha256=U8MwxPlj0ld3o9dvUegcX-9arw8KvtbGUNe_LZUcjMQ,5325
+cobweb/crawlers/base_crawler.py,sha256=G5qMeTDgoBNQ9DGzWuFmMM4C9KJvGWxptLHeSgiWZrI,5181
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
@@ -25,8 +25,8 @@ cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzX
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=qAl05ybL2Jp6KFjHDHVMfmeBHQmDKPtZleHjHAY7LZc,3277
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.14.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.14.dist-info/METADATA,sha256=FoDq1JfDihDH-hEYOfYNpVcJSGHM44yWHLkWj2aUmJQ,1246
-cobweb_launcher-1.1.14.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.14.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.14.dist-info/RECORD,,
+cobweb_launcher-1.1.15.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.15.dist-info/METADATA,sha256=HbjhwAL_1dC-JO4nhJstX3dXG_c5nGweYZ0U9EZWk3w,1246
+cobweb_launcher-1.1.15.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.15.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.15.dist-info/RECORD,,