cobweb-launcher 1.3.1__py3-none-any.whl → 1.3.2__py3-none-any.whl

cobweb/launchers/launcher_pro.py

@@ -1,7 +1,7 @@
 import time
 
-from base import TaskQueue, Decorators
-from schedulers.scheduler_redis import RedisScheduler
+from cobweb.base import TaskQueue, Decorators
+from cobweb.schedulers import RedisScheduler
 from .launcher import Launcher
 
 

cobweb/schedulers/__init__.py

@@ -0,0 +1 @@
+from .scheduler_redis import RedisScheduler

cobweb/schedulers/scheduler_redis.py

@@ -0,0 +1,69 @@
+import threading
+import time
+
+# from cobweb.base import Seed
+from cobweb.db import RedisDB
+
+
+class RedisScheduler:
+
+    def __init__(self, task, project, scheduler_wait_seconds=30):
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._download_key = "{%s:%s}:download" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+        self._db = RedisDB()
+
+        self.scheduler_wait_seconds = scheduler_wait_seconds
+        self.working = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat_key)
+
+    def set_heartbeat(self):
+        return self._db.setex(self._heartbeat_key, 5)
+
+    def schedule(self, key, count):
+        if not self._db.zcount(key, 0, "(1000"):
+            time.sleep(self.scheduler_wait_seconds)
+        else:
+            source = int(time.time())
+            members = self._db.members(key, source, count=count, _min=0, _max="(1000")
+            for member, priority in members:
+                # seed = Seed(member, priority=priority)
+                yield member.decode(), priority
+
+    def insert(self, key, items):
+        if items:
+            self._db.zadd(key, items, nx=True)
+
+    def reset(self, keys, reset_time=30):
+        if self._db.lock(self._reset_lock_key, t=120):
+
+            if isinstance(keys, str):
+                keys = [keys]
+
+            _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
+
+            for key in keys:
+                self._db.members(key, 0, _min=_min, _max="(0")
+
+            if not self.heartbeat:
+                self.working.set()
+                time.sleep(10)
+
+            self._db.delete(self._reset_lock_key)
+
+    def refresh(self, key, items: dict[str, int]):
+        refresh_time = int(time.time())
+        its = {k: -refresh_time - v / 1000 for k, v in items}
+        self._db.zadd(key, item=its, xx=True)
+
+    def delete(self, key, values):
+        self._db.zrem(key, *values)
+
+
+
+

{cobweb_launcher-1.3.1.dist-info → cobweb_launcher-1.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.1
+Version: 1.3.2
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.3.1.dist-info → cobweb_launcher-1.3.2.dist-info}/RECORD

@@ -24,13 +24,15 @@ cobweb/launchers/__init__.py,sha256=uzfPkLbY2m0wsIR_s93VFxmO0U49GgUG7hXPzMYdye0,
 cobweb/launchers/launcher.py,sha256=0BCXzZ05PFB6aQejwVBuMPwoxhlc2BLYL541HO5yiYM,7238
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
 cobweb/launchers/launcher_api.py,sha256=qPazoC7U-UmgebbiTkhl6f4yQmN34XMl6HawekhAhEo,5789
-cobweb/launchers/launcher_pro.py,sha256=OMbo4Lu9yQagoznD3DawCT3hizBTvJBVKu10XfDy7Lw,3411
+cobweb/launchers/launcher_pro.py,sha256=np6cY7UmW7-649mORdsCvbLy2_mkoXhNZ3OLtev555I,3409
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/pipelines/pipeline.py,sha256=FTl2JOoF3ysWa-mxb1pbmayEM4IXQH5a4L5CymlBQNU,1293
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
 cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
+cobweb/schedulers/__init__.py,sha256=WJgBXsPE8zpJQ9L_-bHIUpBcaB2G4HmutDdWF3ud1Bs,44
+cobweb/schedulers/scheduler_redis.py,sha256=Aw7de0sXigRAxJgqUhHWu30hMBzgEWjkj-3OXXqmldg,2118
 cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=fZ-16TAf51dovGIKaHsQthgTkbSEMAteaJPluZhniJI,879
@@ -101,8 +103,8 @@ cobweb_new/utils/__init__.py,sha256=c9macpjc15hrCUCdzO5RR_sgK_B9kvJKreSGprZ1ld4,
 cobweb_new/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb_new/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb_new/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.3.1.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.3.1.dist-info/METADATA,sha256=WiWGFXMCuA4T5UJ3S5gCbpqRyN5Ja6ZmK9lgtOm7BEI,6509
-cobweb_launcher-1.3.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.3.1.dist-info/top_level.txt,sha256=A0GPGeX6QtxXg7AJno3SVRTHtVCCqeRIOrpwDoXg9qs,15
-cobweb_launcher-1.3.1.dist-info/RECORD,,
+cobweb_launcher-1.3.2.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.3.2.dist-info/METADATA,sha256=bnweVxLU9zIGyMqQ7mvKdMvzTYjVTIhJ1ei0tm35SOg,6509
+cobweb_launcher-1.3.2.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.3.2.dist-info/top_level.txt,sha256=A0GPGeX6QtxXg7AJno3SVRTHtVCCqeRIOrpwDoXg9qs,15
+cobweb_launcher-1.3.2.dist-info/RECORD,,