cobweb-launcher 1.2.58__py3-none-any.whl → 1.2.60__py3-none-any.whl

cobweb/db/redis_db.py

@@ -4,7 +4,7 @@ from cobweb import setting
 from redis.exceptions import ConnectionError, TimeoutError
 
 
-class RedisClient:
+class RedisDB:
     def __init__(self, **kwargs):
         redis_config = kwargs or setting.REDIS_CONFIG
         self.host = redis_config['host']
@@ -147,4 +147,69 @@ class RedisClient:
 
     def execute_lua(self, lua_script: str, keys: list, *args):
         execute = self.execute_command("register_script", lua_script)
-        return execute(keys=keys, args=args)
+        return execute(keys=keys, args=args)
+
+    def lock(self, key, t=15) -> bool:
+        lua_script = """
+        local status = redis.call('setnx', KEYS[1], 1)
+        if ( status == 1 ) then
+            redis.call('expire', KEYS[1], ARGV[1])
+        end 
+        return status 
+        """
+        status = self.execute_lua(lua_script, [key], t)
+        return bool(status)
+
+    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
+        lua_script = """
+        local min = ARGV[1]
+        local max = ARGV[2]
+        local start = ARGV[3]
+        local count = ARGV[4]
+        local score = ARGV[5]
+        local members = nil
+
+        if ( type(count) == string ) then
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
+        else
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
+        end
+
+        local result = {}
+
+        for i = 1, #members, 2 do
+            local priority = nil
+            local member = members[i]
+            local originPriority = nil
+            if ( members[i+1] + 0 < 0 ) then
+                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
+            else
+                originPriority = math.floor(members[i+1])
+            end
+
+            if ( score + 0 >= 1000 ) then
+                priority = -score - originPriority / 1000
+            elseif ( score + 0 == 0 ) then
+                priority = originPriority
+            else
+                originPriority = score 
+                priority = score
+            end
+            redis.call('zadd', KEYS[1], priority, member)
+            table.insert(result, member)
+            table.insert(result, originPriority)
+        end
+
+        return result
+        """
+        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
+        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
+
+    def done(self, keys: list, *args) -> list:
+        lua_script = """
+        for i, member in ipairs(ARGV) do
+            redis.call("zrem", KEYS[1], member)
+            redis.call("sadd", KEYS[2], member)
+        end
+        """
+        self.execute_lua(lua_script, keys, *args)

cobweb/launchers/launcher_api.py

@@ -183,8 +183,13 @@ class LauncherApi(Launcher):
                         logger.info("pause! waiting for resume...")
                 elif check_emtpy_times > 2:
                     self.__DOING__ = {}
-                    if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
+                    seed_count = self._db.zcard(self._todo_key)
+                    logger.info(f"队列剩余种子数:{seed_count}")
+                    if not seed_count:
+                        logger.info("Done! pause set...")
                         self._pause.set()
+                    else:
+                        self._pause.clear()
                 else:
                     logger.info(
                         "check whether the task is complete, "
@@ -192,6 +197,8 @@ class LauncherApi(Launcher):
                     )
                     check_emtpy_times += 1
             else:
+                if self._pause.is_set():
+                    self._pause.clear()
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
                     doing_len=len(self.__DOING__.keys()),

cobweb/launchers/launcher_pro.py

@@ -1,7 +1,6 @@
 import time
 import threading
 
-from h5py.h5pl import insert
 
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
@@ -199,6 +198,8 @@ class LauncherPro(Launcher):
                     )
                     check_emtpy_times += 1
             else:
+                if self._pause.is_set():
+                    self._pause.clear()
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
                     doing_len=len(self.__DOING__.keys()),

{cobweb_launcher-1.2.58.dist-info → cobweb_launcher-1.2.60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.58
+Version: 1.2.60
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.58.dist-info → cobweb_launcher-1.2.60.dist-info}/RECORD

@@ -17,15 +17,15 @@ cobweb/crawlers/crawler.py,sha256=pePDGroD6JJAht5QTU51L7MkFIY4ob9TKpznZ2wUmsw,89
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb/db/redis_db.py,sha256=7lnEbPjfyvWuuP2f9hZAfKxpK31eJCoDAB5wfNkds34,5495
+cobweb/db/redis_db.py,sha256=A8J-QhtSt0GWnRVdFFFrkvg3kkCROwjO7y8efcaIRoA,7721
 cobweb/db/redis_db_new.py,sha256=F09LWVjtC2JFdCaKatZ2bAOLKbsnes85_nZRe2dtSIc,4697
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
 cobweb/launchers/launcher.py,sha256=y1a72sANV-oJVpy_BI7BwKy7fY4AnpHqJK3t6czoCts,7896
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_api.py,sha256=7K7Kl3dk7Ung9iRBwhiMrALEJywcR66ie5RIkLQEM-Y,7909
-cobweb/launchers/launcher_pro.py,sha256=EuTsa6ErqpqytdnWhQxHFhO6nXK1V9huvtAAKc_5a6U,8095
+cobweb/launchers/launcher_api.py,sha256=ydx_xrOjGMhGTDdyN3Oxy5EHUvvbvetNj3mCX1GlNsw,8205
+cobweb/launchers/launcher_pro.py,sha256=0Z3E1HA_D-yy_7hD7Q-HkYdy4v4AHYwyrR4i6HErIBo,8147
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -38,8 +38,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.58.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.58.dist-info/METADATA,sha256=QH36iouuBHu1S7g9-57Zek9aYa0CiEu7aIN7YOoNjzs,6510
-cobweb_launcher-1.2.58.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.58.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.58.dist-info/RECORD,,
+cobweb_launcher-1.2.60.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.60.dist-info/METADATA,sha256=9RTAhglArwI9C7Vr_oIxpE-2i7V3gg15RucZJ3r58n4,6510
+cobweb_launcher-1.2.60.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.60.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.60.dist-info/RECORD,,