cobweb-launcher 1.1.13__py3-none-any.whl → 1.1.14__py3-none-any.whl

cobweb/crawlers/base_crawler.py

@@ -32,7 +32,7 @@ class Crawler(threading.Thread):
     @staticmethod
     def request(seed: Seed) -> Union[Request, BaseItem]:
         stream = True if setting.DOWNLOAD_MODEL else False
-        return Request(seed.url, seed, stream=stream, timeout=5)
+        yield Request(seed.url, seed, stream=stream, timeout=5)
 
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
@@ -43,12 +43,27 @@ class Crawler(threading.Thread):
     def parse(item: Response) -> BaseItem:
         pass
 
-    def get(self) -> Seed:
+    def get_seed(self) -> Seed:
         return self.launcher_queue['todo'].pop()
 
+    def distribute(self, item, seed):
+        if isinstance(item, BaseItem):
+            self.upload_queue.push(item)
+        elif isinstance(item, Seed):
+            self.launcher_queue['new'].push(item)
+        elif isinstance(item, str) and item == DealModel.poll:
+            self.launcher_queue['todo'].push(seed)
+        elif isinstance(item, str) and item == DealModel.done:
+            self.launcher_queue['done'].push(seed)
+        elif isinstance(item, str) and item == DealModel.fail:
+            seed.identifier = DealModel.fail
+            self.launcher_queue['done'].push(seed)
+        else:
+            raise TypeError("yield value type error!")
+
     def spider(self):
         while True:
-            seed = self.get()
+            seed = self.get_seed()
 
             if not seed:
                 continue
@@ -58,63 +73,54 @@ class Crawler(threading.Thread):
                 self.launcher_queue['done'].push(seed)
                 continue
 
-            item = self.request(seed)
-
-            if isinstance(item, Request):
-
-                download_iterators = self.download(item)
-
-                if not isgenerator(download_iterators):
-                    raise TypeError("download function isn't a generator")
-
-                seed_detail_log_info = download_log_info(seed.to_dict)
-
-                try:
-                    for it in download_iterators:
-                        if isinstance(it, Response):
-                            response_detail_log_info = download_log_info(it.to_dict)
-                            logger.info(LogTemplate.download_info.format(
-                                detail=seed_detail_log_info, retry=item.seed.params.retry,
-                                priority=item.seed.params.priority,
-                                seed_version=item.seed.params.seed_version,
-                                identifier=item.seed.identifier or "",
-                                status=it.response, response=response_detail_log_info
-                            ))
-                            parse_iterators = self.parse(it)
-                            if not isgenerator(parse_iterators):
-                                raise TypeError("parse function isn't a generator")
-                            for upload_item in parse_iterators:
-                                if not isinstance(upload_item, BaseItem):
-                                    raise TypeError("upload_item isn't BaseItem subclass")
-                                self.upload_queue.push(upload_item)
-                        elif isinstance(it, BaseItem):
-                            self.upload_queue.push(it)
-                        elif isinstance(it, Seed):
-                            self.launcher_queue['new'].push(it)
-                        elif isinstance(it, str) and it == DealModel.poll:
-                            self.launcher_queue['todo'].push(item)
-                            break
-                        elif isinstance(it, str) and it == DealModel.done:
-                            self.launcher_queue['done'].push(seed)
-                            break
-                        elif isinstance(it, str) and it == DealModel.fail:
-                            seed.params.identifier = DealModel.fail
-                            self.launcher_queue['done'].push(seed)
-                            break
-                        else:
-                            raise TypeError("yield value type error!")
-
-                except Exception as e:
-                    logger.info(LogTemplate.download_exception.format(
-                        detail=seed_detail_log_info, retry=seed.params.retry,
-                        priority=seed.params.priority, seed_version=seed.params.seed_version,
-                        identifier=seed.identifier or "", exception=e
-                    ))
-                    seed.params.retry += 1
-                    self.launcher_queue['todo'].push(seed)
-
-            elif isinstance(item, BaseItem):
-                self.upload_queue.push(item)
+            seed_detail_log_info = download_log_info(seed.to_dict)
+
+            request_iterators = self.request(seed)
+
+            if not isgenerator(request_iterators):
+                raise TypeError("request function isn't a generator!")
+
+            for request_item in request_iterators:
+
+                if isinstance(request_item, BaseItem):
+                    self.upload_queue.push(request_item)
+
+                elif isinstance(request_item, Request):
+                    try:
+                        download_iterators = self.download(request_item)
+                        if not isgenerator(download_iterators):
+                            raise TypeError("download function isn't a generator")
+                        for download_item in download_iterators:
+                            if isinstance(download_item, Response):
+                                response_detail_log_info = download_log_info(download_item.to_dict)
+                                logger.info(LogTemplate.download_info.format(
+                                    detail=seed_detail_log_info,
+                                    retry=seed.params.retry,
+                                    priority=seed.params.priority,
+                                    seed_version=seed.params.seed_version,
+                                    identifier=seed.identifier or "",
+                                    status=download_item.response,
+                                    response=response_detail_log_info
+                                ))
+                                parse_iterators = self.parse(download_item)
+                                if not isgenerator(parse_iterators):
+                                    raise TypeError("parse function isn't a generator")
+                                for parse_item in parse_iterators:
+                                    if isinstance(parse_item, Response):
+                                        raise TypeError("upload_item can't be a Response instance")
+                                    self.distribute(parse_item, seed)
+                            else:
+                                self.distribute(download_item, seed)
+                    except Exception as e:
+                        logger.info(LogTemplate.download_exception.format(
+                            detail=seed_detail_log_info,
+                            retry=seed.params.retry,
+                            priority=seed.params.priority,
+                            seed_version=seed.params.seed_version,
+                            identifier=seed.identifier or "", exception=e
+                        ))
+                        seed.params.retry += 1
+                        self.launcher_queue['todo'].push(seed)
 
     def run(self):
         for index in range(self.spider_thread_num):

cobweb/launchers/launcher.py

@@ -93,7 +93,7 @@ class Launcher(threading.Thread):
             @launcher.request
             def request(seed: Seed) -> Union[Request, BaseItem]:
                 ...
-                return Request(seed.url, seed)
+                yield Request(seed.url, seed)
         """
         def decorator(func):
             self.__CUSTOM_FUNC__["request"] = func

cobweb/launchers/launcher_pro.py

@@ -117,7 +117,7 @@ class LauncherPro(Launcher):
                 seed = self.__LAUNCHER_QUEUE__['done'].pop()
                 if not seed:
                     break
-                if seed.params.identifier == DealModel.fail:
+                if seed.identifier == DealModel.fail:
                     f_seeds.append(seed.to_string)
                 elif self._done_model == 1:
                     s_seeds.append(seed.to_string)

{cobweb_launcher-1.1.13.dist-info → cobweb_launcher-1.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.13
+Version: 1.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.13.dist-info → cobweb_launcher-1.1.14.dist-info}/RECORD

@@ -10,23 +10,23 @@ cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=QxlXztWjV8VvcHu-cTzyoDHaDvoX26iyoJKiWk64HnE,2759
 cobweb/crawlers/__init__.py,sha256=Rr3DTjD-abMA1_FYcQJZYNvQvcWMuEVcsIU6duqHrw4,75
-cobweb/crawlers/base_crawler.py,sha256=H-cUfrn79PAI8Z7Xv_NhW-9J0G-SiBU4wtzymzAJMYc,4961
+cobweb/crawlers/base_crawler.py,sha256=U8MwxPlj0ld3o9dvUegcX-9arw8KvtbGUNe_LZUcjMQ,5325
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
-cobweb/launchers/launcher.py,sha256=XSZrrtg9ge3GLFnMY_AHuo8n1-QJSmY7j_XsheQu0rk,5708
-cobweb/launchers/launcher_pro.py,sha256=T4mgSBqFkbU64EF6Fo_oMxzWldD_-l0rC-zFEZxqBQU,6593
+cobweb/launchers/launcher.py,sha256=LQdlpaF4fafEX01_3B9CB6hD-6YMPag6QOGl9rDprNE,5707
+cobweb/launchers/launcher_pro.py,sha256=20o-Yd_JAdzJPchpilCtCd61hrf90LKkxXcCjm2azhQ,6586
 cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=qAl05ybL2Jp6KFjHDHVMfmeBHQmDKPtZleHjHAY7LZc,3277
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.13.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.13.dist-info/METADATA,sha256=ShOTXcWNMBOaLR50TU0QFjR25Ae9aZ-PNyBgwTkxckE,1246
-cobweb_launcher-1.1.13.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.13.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.13.dist-info/RECORD,,
+cobweb_launcher-1.1.14.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.14.dist-info/METADATA,sha256=FoDq1JfDihDH-hEYOfYNpVcJSGHM44yWHLkWj2aUmJQ,1246
+cobweb_launcher-1.1.14.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.14.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.14.dist-info/RECORD,,