cobweb-launcher 0.1.20__tar.gz → 0.1.22__tar.gz

{cobweb-launcher-0.1.20/cobweb_launcher.egg-info → cobweb-launcher-0.1.22}/PKG-INFO

@@ -1,20 +1,17 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.20
+Version: 0.1.22
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
 Author-email: 2604868278@qq.com
 License: MIT
 Keywords: cobweb-launcher, cobweb
+Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests>=2.19.1
-Requires-Dist: oss2>=2.18.1
-Requires-Dist: redis>=4.4.4
-Requires-Dist: aliyun-log-python-sdk
 
 # cobweb
 
@@ -43,3 +40,5 @@ need deal
 
 
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
+
+

{cobweb-launcher-0.1.20 → cobweb-launcher-0.1.22}/cobweb/db/redis_db.py

@@ -119,7 +119,7 @@ class RedisDB:
     def get_seed(self, length: int = 200):
         cs = time.time()
 
-        if self._get_lock(key=self.update_lock, t=Setting.SCHEDULER_LOCK_TIME):
+        if self._get_lock(key=self.update_lock):
 
             update_item, result = {}, []
 

{cobweb-launcher-0.1.20 → cobweb-launcher-0.1.22}/cobweb/equip/single/launcher.py

@@ -171,7 +171,8 @@ def launcher(task):
                 target=spider.spider_task,
                 args=(
                     stop, func, item,
-                    redis_db.del_seed
+                    redis_db.del_seed,
+                    redis_db.add_seed
                 )
             ).start()
 

{cobweb-launcher-0.1.20 → cobweb-launcher-0.1.22}/cobweb/equip/single/models.py

@@ -53,7 +53,7 @@ class Spider:
         self.storage = storage
         self.queue = queue
 
-    def spider_task(self, stop, func, item, del_seed):
+    def spider_task(self, stop, func, item, del_seed, add_seed):
         while not stop.is_set():
 
             seed = self.queue.pop()
@@ -72,6 +72,7 @@ class Spider:
 
                 store_queue = None
                 store_data = list()
+                add_seed_list = list()
                 iterators = func(item, seed)
 
                 if not isgenerator(iterators):
@@ -94,7 +95,7 @@ class Spider:
                             store_queue = it.queue()
                         store_data.append(it.struct_data)
                     elif isinstance(it, Seed):
-                        self.queue.push(it)
+                        add_seed_list.append(it)
 
                     elif isinstance(it, str) and it == DealModel.polling:
                         self.queue.push(seed)
@@ -109,12 +110,13 @@ class Spider:
                         raise TypeError("yield value type error!")
 
                 if not status:
-                    seed._retry += 1
-                    self.queue.push(seed)
-
-                elif store_queue and store_data:
+                    raise Exception("yield value type error!")
+                if store_queue and store_data:
                     store_data.append(seed)
                     store_queue.push(store_data)
+                if add_seed_list:
+                    del_seed(seed, spider_status=True)
+                    add_seed(add_seed_list)
 
             except Exception as e:
                 seed._retry += 1

cobweb-launcher-0.1.22/cobweb/new.py

@@ -0,0 +1,20 @@
+
+
+
+
+class Launcher:
+
+    def __init__(self):
+        pass
+
+    def register(self, task_name, func):
+        pass
+
+    def launch(self, task_name):
+
+        def decorator(func):
+            # 注册爬虫程序
+            self.register(task_name, func)
+            return func
+
+        return decorator

{cobweb-launcher-0.1.20 → cobweb-launcher-0.1.22}/cobweb/task.py

@@ -6,7 +6,6 @@ from .utils import parse_info, struct_start_seeds
 def init_task_env():
     Setting.RESET_SCORE = int(os.getenv("RESET_SCORE", 600))
     Setting.CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
-    Setting.SCHEDULER_LOCK_TIME = int(os.getenv("SCHEDULER_LOCK_TIME", 15))
     Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
     Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
     Setting.SCHEDULER_WAIT_TIME = float(os.getenv("SCHEDULER_WAIT_TIME", 5))

{cobweb-launcher-0.1.20 → cobweb-launcher-0.1.22/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,20 +1,17 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.20
+Version: 0.1.22
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
 Author-email: 2604868278@qq.com
 License: MIT
 Keywords: cobweb-launcher, cobweb
+Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests>=2.19.1
-Requires-Dist: oss2>=2.18.1
-Requires-Dist: redis>=4.4.4
-Requires-Dist: aliyun-log-python-sdk
 
 # cobweb
 
@@ -43,3 +40,5 @@ need deal
 
 
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
+
+

{cobweb-launcher-0.1.20 → cobweb-launcher-0.1.22}/cobweb_launcher.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ cobweb/constant.py
 cobweb/decorators.py
 cobweb/interface.py
 cobweb/log.py
+cobweb/new.py
 cobweb/setting.py
 cobweb/task.py
 cobweb/utils.py

{cobweb-launcher-0.1.20 → cobweb-launcher-0.1.22}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="0.1.20",
+    version="0.1.22",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",