cobweb-launcher 1.3.9__tar.gz → 1.3.10__tar.gz

{cobweb-launcher-1.3.9/cobweb_launcher.egg-info → cobweb-launcher-1.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.9
+Version: 1.3.10
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.9 → cobweb-launcher-1.3.10}/cobweb/base/__init__.py

@@ -59,7 +59,7 @@ class TaskQueue:
                 else:
                     raise TypeError(f"{crawler_func.__name__} function return type isn't supported")
                 TaskQueue.DOT.build(
-                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}:{tk.__class__.__name__}",
                     cost_time=round(time.time() - start_time, 2),
                     **tk.to_dict
                 )

{cobweb-launcher-1.3.9 → cobweb-launcher-1.3.10}/cobweb/launchers/launcher.py

@@ -144,30 +144,30 @@ class Launcher(threading.Thread):
 
     @Decorators.stop
     def _polling(self):
-        if TaskQueue.is_empty():
-            if self.pause.is_set():
-                run_time = int(time.time()) - self.app_time
-                if not self.task_model and run_time > self.before_scheduler_wait_seconds:
-                    logger.info("Done! ready to close thread...")
-                    self.stop.set()
-                elif TaskQueue.TODO.length or TaskQueue.DOWNLOAD.length:
-                    logger.info(f"Recovery {self.task} task run！")
-                    self.check_emtpy_times = 0
-                    self.pause.clear()
-                else:
-                    logger.info("pause! waiting for resume...")
-            elif self.check_emtpy_times > 2:
-                logger.info("pause! waiting for resume...")
-                self.doing_seeds = {}
-                self._task_info['todo'] = {}
-                self._task_info['download'] = {}
-                self.pause.set()
+        time.sleep(10)
+        if self.pause.is_set():
+            run_time = int(time.time()) - self.app_time
+            if not self.task_model and run_time > self.before_scheduler_wait_seconds:
+                logger.info("Done! ready to close thread...")
+                self.stop.set()
+            elif TaskQueue.TODO.length or TaskQueue.DOWNLOAD.length:
+                logger.info(f"Recovery {self.task} task run！")
+                self.check_emtpy_times = 0
+                self.pause.clear()
             else:
-                logger.info(
-                    "check whether the task is complete, "
-                    f"reset times {3 - self.check_emtpy_times}"
-                )
-                self.check_emtpy_times += 1
+                logger.info("pause! waiting for resume...")
+        elif TaskQueue.is_empty() and self.check_emtpy_times > 2:
+            logger.info("pause! waiting for resume...")
+            self.doing_seeds = {}
+            self._task_info['todo'] = {}
+            self._task_info['download'] = {}
+            self.pause.set()
+        elif TaskQueue.is_empty():
+            logger.info(
+                "check whether the task is complete, "
+                f"reset times {3 - self.check_emtpy_times}"
+            )
+            self.check_emtpy_times += 1
         else:
             logger.info(LogTemplate.launcher_polling.format(
                 task=self.task,
@@ -182,7 +182,6 @@ class Launcher(threading.Thread):
                 seed_queue_len=TaskQueue.SEED.length,
                 download_queue_len=TaskQueue.DOWNLOAD.length
             ))
-        time.sleep(10)
 
     def run(self):
         Crawler = dynamic_load_class(self.crawler_path)

{cobweb-launcher-1.3.9 → cobweb-launcher-1.3.10}/cobweb/schedulers/scheduler_api.py

@@ -33,7 +33,7 @@ class ApiScheduler:
             members = self._db.members(key, source, count=count, _min=0, _max="(1000")
             for member, priority in members:
                 # seed = Seed(member, priority=priority)
-                yield member.decode(), priority
+                yield member, priority
 
     def insert(self, key, items):
         if items:
@@ -48,7 +48,8 @@ class ApiScheduler:
             _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
 
             for key in keys:
-                self._db.members(key, 0, _min=_min, _max="(0")
+                if self._db.exists(key):
+                    self._db.members(key, 0, _min=_min, _max="(0")
 
             if not self.heartbeat:
                 self.working.set()
@@ -58,7 +59,7 @@ class ApiScheduler:
 
     def refresh(self, key, items: dict[str, int]):
         refresh_time = int(time.time())
-        its = {k: -refresh_time - v / 1000 for k, v in items}
+        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
         if its:
             self._db.zadd(key, item=its, xx=True)
 

{cobweb-launcher-1.3.9 → cobweb-launcher-1.3.10}/cobweb/schedulers/scheduler_redis.py

@@ -33,7 +33,7 @@ class RedisScheduler:
             members = self._db.members(key, source, count=count, _min=0, _max="(1000")
             for member, priority in members:
                 # seed = Seed(member, priority=priority)
-                yield member.decode(), priority
+                yield member, priority
 
     def insert(self, key, items):
         if items:
@@ -48,7 +48,8 @@ class RedisScheduler:
             _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
 
             for key in keys:
-                self._db.members(key, 0, _min=_min, _max="(0")
+                if self._db.exists(key):
+                    self._db.members(key, 0, _min=_min, _max="(0")
 
             if not self.heartbeat:
                 self.working.set()
@@ -58,7 +59,7 @@ class RedisScheduler:
 
     def refresh(self, key, items: dict[str, int]):
         refresh_time = int(time.time())
-        its = {k: -refresh_time - v / 1000 for k, v in items}
+        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
         if its:
             self._db.zadd(key, item=its, xx=True)
 

{cobweb-launcher-1.3.9 → cobweb-launcher-1.3.10/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.9
+Version: 1.3.10
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.9 → cobweb-launcher-1.3.10}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.3.9",
+    version="1.3.10",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",