cobweb-launcher 1.2.61__tar.gz → 1.2.62__tar.gz

{cobweb-launcher-1.2.61/cobweb_launcher.egg-info → cobweb-launcher-1.2.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.61
+Version: 1.2.62
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.61 → cobweb-launcher-1.2.62}/cobweb/crawlers/crawler.py

@@ -83,7 +83,7 @@ class Crawler(threading.Thread):
         if isinstance(item, BaseItem):
             self._upload_data(item)
         elif isinstance(item, Seed):
-            self._add_seed(item)
+            self._add_seed((seed, item))
         elif isinstance(item, str) and item == DealModel.poll:
             self._set_seed(seed)
         elif isinstance(item, str) and item == DealModel.done:

{cobweb-launcher-1.2.61 → cobweb-launcher-1.2.62}/cobweb/launchers/launcher.py

@@ -173,8 +173,8 @@ class Launcher(threading.Thread):
     def _upload_data(self, data, **kwargs):
         self.__LAUNCHER_QUEUE__["upload"].push(data, **kwargs)
 
-    def _add_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["new"].push(seed, **kwargs)
+    def _add_seed(self, seeds, **kwargs):
+        self.__LAUNCHER_QUEUE__["new"].push(seeds, direct_insertion=True, **kwargs)
 
     def _delete_seed(self, seed, **kwargs):
         self.__LAUNCHER_QUEUE__["done"].push(seed, **kwargs)

{cobweb-launcher-1.2.61 → cobweb-launcher-1.2.62}/cobweb/launchers/launcher_air.py

@@ -13,15 +13,20 @@ class LauncherAir(Launcher):
 
     @check_pause
     def _insert(self):
-        seeds = {}
+        new_seeds = {}
+        del_seeds = set()
         status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
         for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed:
+            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed_tuple:
                 break
-            seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(seeds)
+            seed, new_seed = seed_tuple
+            new_seeds[new_seed.to_string] = new_seed.params.priority
+            del_seeds.add(seed.to_string)
+        if new_seeds:
+            self.__LAUNCHER_QUEUE__['todo'].push(new_seeds)
+        if del_seeds:
+            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

{cobweb-launcher-1.2.61 → cobweb-launcher-1.2.62}/cobweb/launchers/launcher_api.py

@@ -106,14 +106,20 @@ class LauncherApi(Launcher):
         """
         添加新种子到redis队列中
         """
-        seeds = {}
+        new_seeds = {}
+        del_seeds = set()
         status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
         for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if seed:
-                seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self._db.zadd(self._todo_key, seeds, nx=True)
+            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed_tuple:
+                break
+            seed, new_seed = seed_tuple
+            new_seeds[new_seed.to_string] = new_seed.params.priority
+            del_seeds.add(seed.to_string)
+        if new_seeds:
+            self._db.zadd(self._todo_key, new_seeds, nx=True)
+        if del_seeds:
+            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

{cobweb-launcher-1.2.61 → cobweb-launcher-1.2.62}/cobweb/launchers/launcher_pro.py

@@ -108,14 +108,20 @@ class LauncherPro(Launcher):
         """
         添加新种子到redis队列中
         """
-        seeds = {}
+        new_seeds = {}
+        del_seeds = set()
         status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
         for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if seed:
-                seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self._db.zadd(self._todo_key, seeds, nx=True)
+            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed_tuple:
+                break
+            seed, new_seed = seed_tuple
+            new_seeds[new_seed.to_string] = new_seed.params.priority
+            del_seeds.add(seed.to_string)
+        if new_seeds:
+            self._db.zadd(self._todo_key, new_seeds, nx=True)
+        if del_seeds:
+            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

{cobweb-launcher-1.2.61 → cobweb-launcher-1.2.62/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.61
+Version: 1.2.62
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.61 → cobweb-launcher-1.2.62}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.61",
+    version="1.2.62",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",