cobweb-launcher 1.3.8__py3-none-any.whl → 1.3.10__py3-none-any.whl

cobweb/base/__init__.py

@@ -59,7 +59,7 @@ class TaskQueue:
                 else:
                     raise TypeError(f"{crawler_func.__name__} function return type isn't supported")
                 TaskQueue.DOT.build(
-                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}:{tk.__class__.__name__}",
                     cost_time=round(time.time() - start_time, 2),
                     **tk.to_dict
                 )

cobweb/launchers/launcher.py

@@ -144,30 +144,30 @@ class Launcher(threading.Thread):
 
     @Decorators.stop
     def _polling(self):
-        if TaskQueue.is_empty():
-            if self.pause.is_set():
-                run_time = int(time.time()) - self.app_time
-                if not self.task_model and run_time > self.before_scheduler_wait_seconds:
-                    logger.info("Done! ready to close thread...")
-                    self.stop.set()
-                elif TaskQueue.TODO.length or TaskQueue.DOWNLOAD.length:
-                    logger.info(f"Recovery {self.task} task run！")
-                    self.check_emtpy_times = 0
-                    self.pause.clear()
-                else:
-                    logger.info("pause! waiting for resume...")
-            elif self.check_emtpy_times > 2:
-                logger.info("pause! waiting for resume...")
-                self.doing_seeds = {}
-                self._task_info['todo'] = {}
-                self._task_info['download'] = {}
-                self.pause.set()
+        time.sleep(10)
+        if self.pause.is_set():
+            run_time = int(time.time()) - self.app_time
+            if not self.task_model and run_time > self.before_scheduler_wait_seconds:
+                logger.info("Done! ready to close thread...")
+                self.stop.set()
+            elif TaskQueue.TODO.length or TaskQueue.DOWNLOAD.length:
+                logger.info(f"Recovery {self.task} task run！")
+                self.check_emtpy_times = 0
+                self.pause.clear()
             else:
-                logger.info(
-                    "check whether the task is complete, "
-                    f"reset times {3 - self.check_emtpy_times}"
-                )
-                self.check_emtpy_times += 1
+                logger.info("pause! waiting for resume...")
+        elif TaskQueue.is_empty() and self.check_emtpy_times > 2:
+            logger.info("pause! waiting for resume...")
+            self.doing_seeds = {}
+            self._task_info['todo'] = {}
+            self._task_info['download'] = {}
+            self.pause.set()
+        elif TaskQueue.is_empty():
+            logger.info(
+                "check whether the task is complete, "
+                f"reset times {3 - self.check_emtpy_times}"
+            )
+            self.check_emtpy_times += 1
         else:
             logger.info(LogTemplate.launcher_polling.format(
                 task=self.task,
@@ -182,7 +182,6 @@ class Launcher(threading.Thread):
                 seed_queue_len=TaskQueue.SEED.length,
                 download_queue_len=TaskQueue.DOWNLOAD.length
             ))
-        time.sleep(10)
 
     def run(self):
         Crawler = dynamic_load_class(self.crawler_path)

cobweb/schedulers/scheduler_api.py

@@ -33,7 +33,7 @@ class ApiScheduler:
             members = self._db.members(key, source, count=count, _min=0, _max="(1000")
             for member, priority in members:
                 # seed = Seed(member, priority=priority)
-                yield member.decode(), priority
+                yield member, priority
 
     def insert(self, key, items):
         if items:
@@ -48,7 +48,8 @@ class ApiScheduler:
             _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
 
             for key in keys:
-                self._db.members(key, 0, _min=_min, _max="(0")
+                if self._db.exists(key):
+                    self._db.members(key, 0, _min=_min, _max="(0")
 
             if not self.heartbeat:
                 self.working.set()
@@ -58,11 +59,13 @@ class ApiScheduler:
 
     def refresh(self, key, items: dict[str, int]):
         refresh_time = int(time.time())
-        its = {k: -refresh_time - v / 1000 for k, v in items}
-        self._db.zadd(key, item=its, xx=True)
+        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
+        if its:
+            self._db.zadd(key, item=its, xx=True)
 
     def delete(self, key, values):
-        self._db.zrem(key, *values)
+        if values:
+            self._db.zrem(key, *values)
 
 
 

cobweb/schedulers/scheduler_redis.py

@@ -33,7 +33,7 @@ class RedisScheduler:
             members = self._db.members(key, source, count=count, _min=0, _max="(1000")
             for member, priority in members:
                 # seed = Seed(member, priority=priority)
-                yield member.decode(), priority
+                yield member, priority
 
     def insert(self, key, items):
         if items:
@@ -48,7 +48,8 @@ class RedisScheduler:
             _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
 
             for key in keys:
-                self._db.members(key, 0, _min=_min, _max="(0")
+                if self._db.exists(key):
+                    self._db.members(key, 0, _min=_min, _max="(0")
 
             if not self.heartbeat:
                 self.working.set()
@@ -58,11 +59,13 @@ class RedisScheduler:
 
     def refresh(self, key, items: dict[str, int]):
         refresh_time = int(time.time())
-        its = {k: -refresh_time - v / 1000 for k, v in items}
-        self._db.zadd(key, item=its, xx=True)
+        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
+        if its:
+            self._db.zadd(key, item=its, xx=True)
 
     def delete(self, key, values):
-        self._db.zrem(key, *values)
+        if values:
+            self._db.zrem(key, *values)
 
 
 

{cobweb_launcher-1.3.8.dist-info → cobweb_launcher-1.3.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.8
+Version: 1.3.10
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.3.8.dist-info → cobweb_launcher-1.3.10.dist-info}/RECORD

@@ -1,7 +1,7 @@
 cobweb/__init__.py,sha256=oaEfsGUuGP0s39UbFRwrnsjMUeuB6QvQIAwStKFyUTk,83
 cobweb/constant.py,sha256=eofONAntk9O6S-cb4KbYGYHL_u7nBlOqqFOw_HzJHAU,3588
 cobweb/setting.py,sha256=pY6LKsgWI3164GiGA1z_y26LVf5-3mpiEgmm86mKRdY,3135
-cobweb/base/__init__.py,sha256=HHxaDy0x6HKOq5BPhl8eNgqPx06-XYWe-WrM0GQZDDY,5108
+cobweb/base/__init__.py,sha256=HWcjTGKYXCnbmeb70QMqyLwUGxPuxx9IxUJG6PxKpF8,5132
 cobweb/base/basic.py,sha256=s5G4LBZiLUfoymV-gLSIqeH-OJ7q7-L35sBa6xEH3EI,7666
 cobweb/base/common_queue.py,sha256=Gor7sR3h1hlZWaI0XcNAbf0S15Ftjr3DFRWNTGL13uU,1137
 cobweb/base/dotting.py,sha256=lfFXXqnVP__hxlW3qH5Bnuq69KtnFaQLbcz1M8e2Ajg,1239
@@ -18,7 +18,7 @@ cobweb/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=m_XNG2bWuMbirPt3d0_s-Ezl1xycfUxeqZnwq_kkfuo,116
-cobweb/launchers/launcher.py,sha256=NFwpc_0Um0hbDm1A8glWA4fcW6mNYL1eon4t3JAQUlw,7411
+cobweb/launchers/launcher.py,sha256=O7HjZ2lewsZ5IIiZ2vhCAqGidJ1JKMxS5fs4KlslBkA,7332
 cobweb/launchers/launcher_air.py,sha256=yPr395HVIIHAq6lqRcYJu7c0KkfO9V8O-2sn0hC96p0,2990
 cobweb/launchers/launcher_api.py,sha256=TfLrLXazFWsOJLI7caMGfZozCttL1WTwTo3uUpN_FV0,3370
 cobweb/launchers/launcher_pro.py,sha256=2H-TcvQx-ga78GLNTa-GXMLYAj9nEeCJSWf8xl-1ISQ,3374
@@ -27,14 +27,14 @@ cobweb/pipelines/pipeline.py,sha256=Pycm22bHId9a3gdP81D5y7SsuMndYooTb5n4zQxP7dM,
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
 cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
 cobweb/schedulers/__init__.py,sha256=y7Lv_7b0zfTl0OhIONb_8u1K1C9gVlBA-xz_XG_kI9g,85
-cobweb/schedulers/scheduler_api.py,sha256=mC54QOS0PEu4SFvxfD5Qr9239hAxwMrKTg-33rirANE,2112
-cobweb/schedulers/scheduler_redis.py,sha256=Aw7de0sXigRAxJgqUhHWu30hMBzgEWjkj-3OXXqmldg,2118
+cobweb/schedulers/scheduler_api.py,sha256=pFEdS1H4zuzxwMhCV-G7CoLz-rEOPv4EVo3xZUXTyDo,2199
+cobweb/schedulers/scheduler_redis.py,sha256=E5fjc3nNld8GbUhUGT7uY4smRejj2J2ZIzp2g6lhxFM,2205
 cobweb/utils/__init__.py,sha256=YvD4mIDBd9jmGA6WJBcwkgDU2jRFNBCEbarZCSUBAHE,114
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/oss.py,sha256=6Qlhdde7CcwD69bBe2rGWHY3-aptG9NXB_DZLhjgDRQ,3553
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.3.8.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.3.8.dist-info/METADATA,sha256=vxO6miknf7w1rom7AF3FubCwBiiYCHgS5xXVGKNdQdk,6509
-cobweb_launcher-1.3.8.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.3.8.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.3.8.dist-info/RECORD,,
+cobweb_launcher-1.3.10.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.3.10.dist-info/METADATA,sha256=_OKrcjWsAAL5b8PmzTHSTMo0xj3gpxv0aHFXSbVdAgc,6510
+cobweb_launcher-1.3.10.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.3.10.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.3.10.dist-info/RECORD,,