cobweb-launcher 1.1.18__tar.gz → 1.1.20__tar.gz

{cobweb-launcher-1.1.18/cobweb_launcher.egg-info → cobweb-launcher-1.1.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.18
+Version: 1.1.20
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.18 → cobweb-launcher-1.1.20}/cobweb/crawlers/base_crawler.py

@@ -81,17 +81,22 @@ class Crawler(threading.Thread):
                 if not isgenerator(request_iterators):
                     raise TypeError("request function isn't a generator!")
 
+                iterator_status = False
+
                 for request_item in request_iterators:
 
-                    if isinstance(request_item, BaseItem):
-                        self.upload_queue.push(request_item)
+                    iterator_status = True
 
-                    elif isinstance(request_item, Request):
+                    if isinstance(request_item, Request):
+                        iterator_status = False
                         download_iterators = self.download(request_item)
                         if not isgenerator(download_iterators):
                             raise TypeError("download function isn't a generator")
+
                         for download_item in download_iterators:
+                            iterator_status = True
                             if isinstance(download_item, Response):
+                                iterator_status = False
                                 logger.info(LogTemplate.download_info.format(
                                     detail=seed_detail_log_info,
                                     retry=seed.params.retry,
@@ -105,11 +110,18 @@ class Crawler(threading.Thread):
                                 if not isgenerator(parse_iterators):
                                     raise TypeError("parse function isn't a generator")
                                 for parse_item in parse_iterators:
+                                    iterator_status = True
                                     if isinstance(parse_item, Response):
                                         raise TypeError("upload_item can't be a Response instance")
                                     self.distribute(parse_item, seed)
                             else:
                                 self.distribute(download_item, seed)
+                    else:
+                        self.distribute(request_item, seed)
+
+                if not iterator_status:
+                    raise ValueError("request/download/parse function yield value error!")
+
             except Exception as e:
                 logger.info(LogTemplate.download_exception.format(
                     detail=seed_detail_log_info,

{cobweb-launcher-1.1.18 → cobweb-launcher-1.1.20/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.18
+Version: 1.1.20
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.18 → cobweb-launcher-1.1.20}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.1.18",
+    version="1.1.20",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",