cobweb-launcher 1.1.14__tar.gz → 1.1.16__tar.gz

{cobweb-launcher-1.1.14/cobweb_launcher.egg-info → cobweb-launcher-1.1.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.14
+Version: 1.1.16
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.14 → cobweb-launcher-1.1.16}/cobweb/crawlers/base_crawler.py

@@ -69,30 +69,29 @@ class Crawler(threading.Thread):
                 continue
 
             elif seed.params.retry >= self.max_retries:
-                seed.params.identifier = DealModel.fail
+                seed.identifier = DealModel.fail
                 self.launcher_queue['done'].push(seed)
                 continue
 
             seed_detail_log_info = download_log_info(seed.to_dict)
 
-            request_iterators = self.request(seed)
+            try:
+                request_iterators = self.request(seed)
 
-            if not isgenerator(request_iterators):
-                raise TypeError("request function isn't a generator!")
+                if not isgenerator(request_iterators):
+                    raise TypeError("request function isn't a generator!")
 
-            for request_item in request_iterators:
+                for request_item in request_iterators:
 
-                if isinstance(request_item, BaseItem):
-                    self.upload_queue.push(request_item)
+                    if isinstance(request_item, BaseItem):
+                        self.upload_queue.push(request_item)
 
-                elif isinstance(request_item, Request):
-                    try:
+                    elif isinstance(request_item, Request):
                         download_iterators = self.download(request_item)
                         if not isgenerator(download_iterators):
                             raise TypeError("download function isn't a generator")
                         for download_item in download_iterators:
                             if isinstance(download_item, Response):
-                                response_detail_log_info = download_log_info(download_item.to_dict)
                                 logger.info(LogTemplate.download_info.format(
                                     detail=seed_detail_log_info,
                                     retry=seed.params.retry,
@@ -100,7 +99,7 @@ class Crawler(threading.Thread):
                                     seed_version=seed.params.seed_version,
                                     identifier=seed.identifier or "",
                                     status=download_item.response,
-                                    response=response_detail_log_info
+                                    response=download_log_info(download_item.to_dict)
                                 ))
                                 parse_iterators = self.parse(download_item)
                                 if not isgenerator(parse_iterators):
@@ -111,16 +110,16 @@ class Crawler(threading.Thread):
                                     self.distribute(parse_item, seed)
                             else:
                                 self.distribute(download_item, seed)
-                    except Exception as e:
-                        logger.info(LogTemplate.download_exception.format(
-                            detail=seed_detail_log_info,
-                            retry=seed.params.retry,
-                            priority=seed.params.priority,
-                            seed_version=seed.params.seed_version,
-                            identifier=seed.identifier or "", exception=e
-                        ))
-                        seed.params.retry += 1
-                        self.launcher_queue['todo'].push(seed)
+            except Exception as e:
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info,
+                    retry=seed.params.retry,
+                    priority=seed.params.priority,
+                    seed_version=seed.params.seed_version,
+                    identifier=seed.identifier or "", exception=e
+                ))
+                seed.params.retry += 1
+                self.launcher_queue['todo'].push(seed)
 
     def run(self):
         for index in range(self.spider_thread_num):

{cobweb-launcher-1.1.14 → cobweb-launcher-1.1.16/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.14
+Version: 1.1.16
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.14 → cobweb-launcher-1.1.16}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.1.14",
+    version="1.1.16",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",