cobweb-launcher 1.3.14__py3-none-any.whl → 3.1.0__py3-none-any.whl

cobweb/launchers/launcher_pro.py

@@ -1,88 +0,0 @@
-import time
-
-from cobweb.base import TaskQueue, Decorators, Seed, Request
-from cobweb.schedulers import RedisScheduler
-from .launcher import Launcher
-
-
-class LauncherPro(Launcher):
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__(task, project, custom_setting, **kwargs)
-        self._redis_download = "{%s:%s}:download" % (project, task)
-        self._redis_todo = "{%s:%s}:todo" % (project, task)
-        self._scheduler = RedisScheduler(task, project)
-
-    @Decorators.stop
-    def _schedule(self):
-        thread_sleep = self.scheduling_wait_time
-        for q, key, size, item_info, Cls in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"], Seed),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"], Request),
-        ]:
-            if q.length < size:
-                for member, priority in self._scheduler.schedule(key, self.scheduling_size):
-                    q.push(Cls(member, priority=priority))
-                    self.add_working_item(key.split(":")[-1], member, priority)
-                thread_sleep = 0.1
-        time.sleep(thread_sleep)
-
-    @Decorators.stop
-    def _heartbeat(self):
-        if self._scheduler.working.is_set():
-            self._scheduler.set_heartbeat()
-        time.sleep(3)
-
-    @Decorators.stop
-    def _reset(self):
-        self._scheduler.reset(
-            keys=[self._redis_todo, self._redis_download],
-            reset_time=self.seed_reset_seconds
-        )
-        time.sleep(30)
-
-    @Decorators.pause
-    def _insert(self):
-        thread_sleep = 0.1
-        for q, key, size in [
-            (TaskQueue.SEED, self._redis_todo, self.seed_queue_size),
-            (TaskQueue.REQUEST, self._redis_download, self.request_queue_size),
-        ]:
-            item_info = {}
-            while (item := q.pop()) and len(item_info.keys()) < self.inserting_size:
-                item_info[item.seed] = item.params.priority
-            if q.length >= size:
-                thread_sleep = self.inserting_wait_time
-            self._scheduler.insert(key, item_info)
-        time.sleep(thread_sleep)
-
-    @Decorators.pause
-    def _refresh(self):
-        self._scheduler.refresh(self._redis_todo, self._task_info["todo"])
-        self._scheduler.refresh(self._redis_download, self._task_info["download"])
-        time.sleep(10)
-
-    @Decorators.pause
-    def _remove(self):
-        thread_sleep = self.removing_wait_time
-        for q, key, size in [
-            (TaskQueue.DELETE, self._redis_todo, self.delete_queue_size),
-            (TaskQueue.DONE, self._redis_download, self.done_queue_size),
-        ]:
-            items = []
-            while (item := q.pop()) and len(items) < self.removing_size:
-                items.append(item)
-            self._scheduler.delete(key, items)
-            self.remove_working_items(key.split(":")[-1], items)
-            if q.length >= size:
-                thread_sleep = 0.1
-        time.sleep(thread_sleep)
-
-    def _init_schedule_thread(self):
-        self._add_thread(func=self._heartbeat)
-        self._add_thread(func=self._reset)
-        self._add_thread(func=self._refresh)
-        self._add_thread(func=self._schedule)
-        self._add_thread(func=self._insert)
-        self._add_thread(func=self._remove)
-        # self._add_thread(func=self._polling)

cobweb/schedulers/scheduler_api.py

@@ -1,72 +0,0 @@
-import threading
-import time
-
-# from cobweb.base import Seed
-from cobweb.db import ApiDB
-
-
-class ApiScheduler:
-
-    def __init__(self, task, project, scheduler_wait_seconds=30):
-        self._todo_key = "{%s:%s}:todo" % (project, task)
-        self._download_key = "{%s:%s}:download" % (project, task)
-        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
-        self._speed_control_key = "speed_control:%s_%s" % (project, task)
-        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
-        self._db = ApiDB()
-
-        self.scheduler_wait_seconds = scheduler_wait_seconds
-        self.working = threading.Event()
-
-    @property
-    def heartbeat(self):
-        return self._db.exists(self._heartbeat_key)
-
-    def set_heartbeat(self):
-        return self._db.setex(self._heartbeat_key, 5)
-
-    def schedule(self, key, count):
-        if not self._db.zcount(key, 0, "(1000"):
-            time.sleep(self.scheduler_wait_seconds)
-        else:
-            source = int(time.time())
-            members = self._db.members(key, source, count=count, _min=0, _max="(1000")
-            for member, priority in members:
-                # seed = Seed(member, priority=priority)
-                yield member, priority
-
-    def insert(self, key, items):
-        if items:
-            self._db.zadd(key, items, nx=True)
-
-    def reset(self, keys, reset_time=30):
-        if self._db.lock(self._reset_lock_key, t=120):
-
-            if isinstance(keys, str):
-                keys = [keys]
-
-            _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
-
-            for key in keys:
-                if self._db.exists(key):
-                    self._db.members(key, 0, _min=_min, _max="(0")
-
-            if not self.heartbeat:
-                self.working.set()
-                time.sleep(10)
-
-            self._db.delete(self._reset_lock_key)
-
-    def refresh(self, key, items: dict[str, int]):
-        refresh_time = int(time.time())
-        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
-        if its:
-            self._db.zadd(key, item=its, xx=True)
-
-    def delete(self, key, values):
-        if values:
-            self._db.zrem(key, *values)
-
-
-
-

cobweb/schedulers/scheduler_redis.py

@@ -1,72 +0,0 @@
-import threading
-import time
-
-# from cobweb.base import Seed
-from cobweb.db import RedisDB
-
-
-class RedisScheduler:
-
-    def __init__(self, task, project, scheduler_wait_seconds=30):
-        self._todo_key = "{%s:%s}:todo" % (project, task)
-        self._download_key = "{%s:%s}:download" % (project, task)
-        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
-        self._speed_control_key = "speed_control:%s_%s" % (project, task)
-        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
-        self._db = RedisDB()
-
-        self.scheduler_wait_seconds = scheduler_wait_seconds
-        self.working = threading.Event()
-
-    @property
-    def heartbeat(self):
-        return self._db.exists(self._heartbeat_key)
-
-    def set_heartbeat(self):
-        return self._db.setex(self._heartbeat_key, 5)
-
-    def schedule(self, key, count):
-        if not self._db.zcount(key, 0, "(1000"):
-            time.sleep(self.scheduler_wait_seconds)
-        else:
-            source = int(time.time())
-            members = self._db.members(key, source, count=count, _min=0, _max="(1000")
-            for member, priority in members:
-                # seed = Seed(member, priority=priority)
-                yield member, priority
-
-    def insert(self, key, items):
-        if items:
-            self._db.zadd(key, items, nx=True)
-
-    def reset(self, keys, reset_time=30):
-        if self._db.lock(self._reset_lock_key, t=120):
-
-            if isinstance(keys, str):
-                keys = [keys]
-
-            _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
-
-            for key in keys:
-                if self._db.exists(key):
-                    self._db.members(key, 0, _min=_min, _max="(0")
-
-            if not self.heartbeat:
-                self.working.set()
-                time.sleep(10)
-
-            self._db.delete(self._reset_lock_key)
-
-    def refresh(self, key, items: dict[str, int]):
-        refresh_time = int(time.time())
-        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
-        if its:
-            self._db.zadd(key, item=its, xx=True)
-
-    def delete(self, key, values):
-        if values:
-            self._db.zrem(key, *values)
-
-
-
-

cobweb_launcher-1.3.14.dist-info/RECORD

@@ -1,40 +0,0 @@
-cobweb/__init__.py,sha256=oaEfsGUuGP0s39UbFRwrnsjMUeuB6QvQIAwStKFyUTk,83
-cobweb/constant.py,sha256=eofONAntk9O6S-cb4KbYGYHL_u7nBlOqqFOw_HzJHAU,3588
-cobweb/setting.py,sha256=pY6LKsgWI3164GiGA1z_y26LVf5-3mpiEgmm86mKRdY,3135
-cobweb/base/__init__.py,sha256=CgNg7BK8uPICSWrLI9Bi6vNQaquBNY2H31TrDo9-fTI,5245
-cobweb/base/basic.py,sha256=eOSHnZT2xR-sOND8J4M3iCJJJUV51QiFi8Yn8JxV7s4,7670
-cobweb/base/common_queue.py,sha256=Gor7sR3h1hlZWaI0XcNAbf0S15Ftjr3DFRWNTGL13uU,1137
-cobweb/base/dotting.py,sha256=lfFXXqnVP__hxlW3qH5Bnuq69KtnFaQLbcz1M8e2Ajg,1239
-cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
-cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
-cobweb/base/request.py,sha256=acGm3OzxsPed5VUTk7D9eeHZPMh7KUNQRUv44G5znZg,2659
-cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
-cobweb/base/seed.py,sha256=PN5J4gKPEXylwyQeSGOBfauxHktxFr7RJe8nVX1hBw4,2987
-cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
-cobweb/crawlers/crawler.py,sha256=A7_qVGXHiULk7PsYIxmYvVCyILdWy4Er8_qmbVDzFzE,3950
-cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
-cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
-cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb/launchers/__init__.py,sha256=m_XNG2bWuMbirPt3d0_s-Ezl1xycfUxeqZnwq_kkfuo,116
-cobweb/launchers/launcher.py,sha256=KB7aL38T3uMh1s78HyHLcS0DS8ovx7VuW6JHn5ooec8,7807
-cobweb/launchers/launcher_air.py,sha256=yPr395HVIIHAq6lqRcYJu7c0KkfO9V8O-2sn0hC96p0,2990
-cobweb/launchers/launcher_api.py,sha256=vpwFxB1azgBk1bS7VhX3jOprQS8fl6Iu_5M-Y3QT67A,3394
-cobweb/launchers/launcher_pro.py,sha256=Kl64CQLcUwW9FeaKFg0GIK51fLZaWkoQlQ346zEvJEE,3398
-cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
-cobweb/pipelines/pipeline.py,sha256=FHY7ZHNZgx-AdbHt1MKHm-w0aigxvyXFV10T0NMdrOE,1381
-cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
-cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
-cobweb/schedulers/__init__.py,sha256=y7Lv_7b0zfTl0OhIONb_8u1K1C9gVlBA-xz_XG_kI9g,85
-cobweb/schedulers/scheduler_api.py,sha256=pFEdS1H4zuzxwMhCV-G7CoLz-rEOPv4EVo3xZUXTyDo,2199
-cobweb/schedulers/scheduler_redis.py,sha256=E5fjc3nNld8GbUhUGT7uY4smRejj2J2ZIzp2g6lhxFM,2205
-cobweb/utils/__init__.py,sha256=YvD4mIDBd9jmGA6WJBcwkgDU2jRFNBCEbarZCSUBAHE,114
-cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
-cobweb/utils/oss.py,sha256=6Qlhdde7CcwD69bBe2rGWHY3-aptG9NXB_DZLhjgDRQ,3553
-cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.3.14.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.3.14.dist-info/METADATA,sha256=l2r1AYRaNcYTFGxPQoqXTo-5SNt470NZRxzpj_QSj_o,6510
-cobweb_launcher-1.3.14.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.3.14.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.3.14.dist-info/RECORD,,