cobweb-launcher 1.1.5__py3-none-any.whl → 1.1.9__py3-none-any.whl

cobweb/crawlers/file_crawler.py

@@ -123,12 +123,14 @@ class FileCrawlerPro(Crawler):
                         item.seed, response, filter=True, msg=f"response content type is {content_type}",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
+                    response.close()
                 elif position == 1 and min_upload_size >= content_length > 0:
                     """过小文件标识返回"""
                     yield Response(
                         item.seed, response, filter=True, msg="file size is too small",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
+                    response.close()
                 elif position == 1 and chunk_size > content_length > min_upload_size:
                     """小文件直接下载"""
                     for part_data in response.iter_content(chunk_size):
@@ -153,12 +155,12 @@ class FileCrawlerPro(Crawler):
                             seed_dict['position'] = position
                             seed_dict['start'] = upload_content_length
 
-                    response.close()
                     if content:
                         oss_util.put_part(key, upload_id, position, content)
                         content_length += len(content)
                     oss_util.merge(key, upload_id)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                    response.close()
 
             elif item.seed.params.identifier == "merge":
                 oss_util.merge(key, seed_dict["upload_id"])

cobweb/launchers/launcher.py

@@ -34,7 +34,7 @@ class Launcher(threading.Thread):
         "_delete",
     ]
 
-    def __init__(self, task, project, custom_setting=None):
+    def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__()
         self.task = task
         self.project = project
@@ -42,10 +42,11 @@ class Launcher(threading.Thread):
         self._stop = threading.Event()  # 结束事件
         self._pause = threading.Event()  # 暂停事件
 
+        _setting = dict()
+
         if custom_setting:
-            setting_ = dict()
             if isinstance(custom_setting, dict):
-                setting_ = custom_setting
+                _setting = custom_setting
             else:
                 if isinstance(custom_setting, str):
                     custom_setting = importlib.import_module(custom_setting)
@@ -53,9 +54,12 @@ class Launcher(threading.Thread):
                     raise Exception
                 for k, v in custom_setting.__dict__.items():
                     if not k.startswith("__") and not inspect.ismodule(v):
-                        setting_[k] = v
-            for k, v in setting_.items():
-                setattr(setting, k, v)
+                        _setting[k] = v
+
+        _setting.update(**kwargs)
+
+        for k, v in _setting.items():
+            setattr(setting, k, v)
 
         self._Crawler = dynamic_load_class(setting.CRAWLER)
         self._Pipeline = dynamic_load_class(setting.PIPELINE)

cobweb/launchers/launcher_pro.py

@@ -9,8 +9,8 @@ from cobweb.constant import DealModel, LogTemplate
 
 class LauncherPro(Launcher):
 
-    def __init__(self, task, project, custom_setting=None):
-        super().__init__(task, project, custom_setting)
+    def __init__(self, task, project, custom_setting=None, **kwargs):
+        super().__init__(task, project, custom_setting, **kwargs)
         self._todo = "{%s:%s}:todo" % (project, task)
         self._done = "{%s:%s}:done" % (project, task)
         self._fail = "{%s:%s}:fail" % (project, task)
@@ -140,22 +140,23 @@ class LauncherPro(Launcher):
         while not self._stop.is_set():
             queue_not_empty_count = 0
             pooling_wait_seconds = 30
-            if not self._db.zcard(self._todo):
-                for q in self.__LAUNCHER_QUEUE__.values():
-                    if q.length != 0:
-                        queue_not_empty_count += 1
-                if self._pause.is_set():
-                    self._pause.clear()
-                    self._execute()
-                elif queue_not_empty_count == 0:
-                    pooling_wait_seconds = 3
-                    check_emtpy_times += 1
-                else:
-                    check_emtpy_times = 0
-                if check_emtpy_times > 2:
-                    check_emtpy_times = 0
-                    self.__DOING__ = {}
-                    self._pause.set()
+            for q in self.__LAUNCHER_QUEUE__.values():
+                if q.length != 0:
+                    queue_not_empty_count += 1
+            if self._pause.is_set():
+                self._pause.clear()
+                self._execute()
+            elif queue_not_empty_count == 0:
+                pooling_wait_seconds = 5
+                check_emtpy_times += 1
+            else:
+                check_emtpy_times = 0
+
+            if not self._db.zcount(self._todo, _min=0, _max="(1000") and check_emtpy_times > 2:
+                check_emtpy_times = 0
+                self.__DOING__ = {}
+                self._pause.set()
+
             if not self._pause.is_set():
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,

{cobweb_launcher-1.1.5.dist-info → cobweb_launcher-1.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.5
+Version: 1.1.9
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.5.dist-info → cobweb_launcher-1.1.9.dist-info}/RECORD

@@ -11,22 +11,22 @@ cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
 cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
 cobweb/crawlers/__init__.py,sha256=_HAXBg7Sq8fsDGSjDm3AQz9aQtLZONpt5b8dSe607mI,91
 cobweb/crawlers/base_crawler.py,sha256=uR1wQ2sJpFovNoAK52293rF03O-jNbv24P5QoNt1tW0,5169
-cobweb/crawlers/file_crawler.py,sha256=LTiHaxhEiJyiAGgodO3an8AYf_y88AeMoFcKae3Vx_M,8381
+cobweb/crawlers/file_crawler.py,sha256=Lr0-eQJlQcgHaojOM5-awcIAJOUi9eZ_E1XOTYuQM28,8455
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
-cobweb/launchers/launcher.py,sha256=O6Kkvqk-0kOxJom8YO9zW18e_2eMYrA5RTS9Xy4TW5k,5665
-cobweb/launchers/launcher_pro.py,sha256=3B7JjT2wk-q_Q9yCIw36OIS0eGVNBFRV5F6WLzH4FKo,6620
+cobweb/launchers/launcher.py,sha256=s4PSfcjYiMiboSB_yEOS68NSMBBE7VW5BY3WzPLxq1w,5700
+cobweb/launchers/launcher_pro.py,sha256=T4mgSBqFkbU64EF6Fo_oMxzWldD_-l0rC-zFEZxqBQU,6593
 cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=uD5aN2oVYImit3amE6TjxWMaTAcbAh9dCnpIQhf4M9Q,3238
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.5.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.5.dist-info/METADATA,sha256=F5vvm0pngBeivTOVQ_srp-7epa3gHadaovtottKAIko,1245
-cobweb_launcher-1.1.5.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.5.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.5.dist-info/RECORD,,
+cobweb_launcher-1.1.9.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.9.dist-info/METADATA,sha256=BmlL5uOSiwZxO-aeZXHrmAjc_DV0rnK2HljaASlBE8w,1245
+cobweb_launcher-1.1.9.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.9.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.9.dist-info/RECORD,,