cobweb-launcher 1.3.1__tar.gz → 1.3.2__tar.gz

{cobweb-launcher-1.3.1/cobweb_launcher.egg-info → cobweb-launcher-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.1
+Version: 1.3.2
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.1 → cobweb-launcher-1.3.2}/cobweb/launchers/launcher_pro.py

@@ -1,7 +1,7 @@
 import time
 
-from base import TaskQueue, Decorators
-from schedulers.scheduler_redis import RedisScheduler
+from cobweb.base import TaskQueue, Decorators
+from cobweb.schedulers import RedisScheduler
 from .launcher import Launcher
 
 

cobweb-launcher-1.3.2/cobweb/schedulers/__init__.py

@@ -0,0 +1 @@
+from .scheduler_redis import RedisScheduler

cobweb-launcher-1.3.2/cobweb/schedulers/scheduler_redis.py

@@ -0,0 +1,69 @@
+import threading
+import time
+
+# from cobweb.base import Seed
+from cobweb.db import RedisDB
+
+
+class RedisScheduler:
+
+    def __init__(self, task, project, scheduler_wait_seconds=30):
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._download_key = "{%s:%s}:download" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+        self._db = RedisDB()
+
+        self.scheduler_wait_seconds = scheduler_wait_seconds
+        self.working = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat_key)
+
+    def set_heartbeat(self):
+        return self._db.setex(self._heartbeat_key, 5)
+
+    def schedule(self, key, count):
+        if not self._db.zcount(key, 0, "(1000"):
+            time.sleep(self.scheduler_wait_seconds)
+        else:
+            source = int(time.time())
+            members = self._db.members(key, source, count=count, _min=0, _max="(1000")
+            for member, priority in members:
+                # seed = Seed(member, priority=priority)
+                yield member.decode(), priority
+
+    def insert(self, key, items):
+        if items:
+            self._db.zadd(key, items, nx=True)
+
+    def reset(self, keys, reset_time=30):
+        if self._db.lock(self._reset_lock_key, t=120):
+
+            if isinstance(keys, str):
+                keys = [keys]
+
+            _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
+
+            for key in keys:
+                self._db.members(key, 0, _min=_min, _max="(0")
+
+            if not self.heartbeat:
+                self.working.set()
+                time.sleep(10)
+
+            self._db.delete(self._reset_lock_key)
+
+    def refresh(self, key, items: dict[str, int]):
+        refresh_time = int(time.time())
+        its = {k: -refresh_time - v / 1000 for k, v in items}
+        self._db.zadd(key, item=its, xx=True)
+
+    def delete(self, key, values):
+        self._db.zrem(key, *values)
+
+
+
+

{cobweb-launcher-1.3.1 → cobweb-launcher-1.3.2/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.1
+Version: 1.3.2
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.1 → cobweb-launcher-1.3.2}/cobweb_launcher.egg-info/SOURCES.txt

@@ -29,6 +29,8 @@ cobweb/pipelines/__init__.py
 cobweb/pipelines/pipeline.py
 cobweb/pipelines/pipeline_console.py
 cobweb/pipelines/pipeline_loghub.py
+cobweb/schedulers/__init__.py
+cobweb/schedulers/scheduler_redis.py
 cobweb/utils/__init__.py
 cobweb/utils/bloom.py
 cobweb/utils/oss.py

{cobweb-launcher-1.3.1 → cobweb-launcher-1.3.2}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.3.1",
+    version="1.3.2",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",