cobweb-launcher 1.2.21__py3-none-any.whl → 1.2.23__py3-none-any.whl

cobweb/db/api_db.py

@@ -1,5 +1,5 @@
-import json
 import os
+import json
 import requests
 
 
@@ -9,19 +9,25 @@ class ApiDB:
         self.host = host or os.getenv("REDIS_API_HOST", "http://127.0.0.1:4396")
 
     def _get_response(self, api, params: dict = None):
-        url = self.host + api
-        response = requests.get(url, params=params)
-        json_data = response.json()
-        response.close()
-        return json_data["data"]
+        try:
+            url = self.host + api
+            response = requests.get(url, params=params)
+            json_data = response.json()
+            response.close()
+            return json_data["data"]
+        except:
+            return None
 
     def _post_response(self, api, params: dict = None, data: dict = None):
-        url = self.host + api
-        headers = {"Content-Type": "application/json"}
-        response = requests.post(url, headers=headers, params=params, data=json.dumps(data))
-        json_data = response.json()
-        response.close()
-        return json_data["data"]
+        try:
+            url = self.host + api
+            headers = {"Content-Type": "application/json"}
+            response = requests.post(url, headers=headers, params=params, data=json.dumps(data))
+            json_data = response.json()
+            response.close()
+            return json_data["data"]
+        except:
+            return None
 
     def get(self, name):
         return self._get_response(api="/get", params=dict(name=name))

cobweb/launchers/launcher_api.py

@@ -73,11 +73,12 @@ class LauncherApi(Launcher):
                 if self.heartbeat else "-inf"
 
             self._db.members(self._todo_key, 0, _min=_min, _max="(0")
-            self._db.delete(self._reset_lock_key)
 
             if not self.heartbeat:
                 self._heartbeat_start_event.set()
 
+            self._db.delete(self._reset_lock_key)
+
         time.sleep(reset_wait_seconds)
 
     @check_pause
@@ -149,9 +150,9 @@ class LauncherApi(Launcher):
 
         if seed_info["count"]:
 
-            succeed_count = self._db.zrem(self._todo_key, *seed_info["common"])
-            succeed_count += self._db.done([self._todo_key, self._done_key], *seed_info["succeed"])
-            failed_count = self._db.done([self._todo_key, self._fail_key], *seed_info["failed"])
+            succeed_count = int(self._db.zrem(self._todo_key, *seed_info["common"]) or 0)
+            succeed_count += int(self._db.done([self._todo_key, self._done_key], *seed_info["succeed"]) or 0)
+            failed_count = int(self._db.done([self._todo_key, self._fail_key], *seed_info["failed"]) or 0)
 
             if failed_count:
                 self.statistics(self._statistics_fail_key, failed_count)
@@ -208,9 +209,9 @@ class LauncherApi(Launcher):
                     doing_len=len(self.__DOING__.keys()),
                     todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
                     done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    redis_seed_count=self._db.zcount(self._todo_key, "-inf", "+inf"),
-                    redis_todo_len=self._db.zcount(self._todo_key, 0, "(1000"),
-                    redis_doing_len=self._db.zcount(self._todo_key, "-inf", "(0"),
+                    redis_seed_count=self._db.zcount(self._todo_key, "-inf", "+inf") or "error",
+                    redis_todo_len=self._db.zcount(self._todo_key, 0, "(1000") or "error",
+                    redis_doing_len=self._db.zcount(self._todo_key, "-inf", "(0") or "error",
                     upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
                 ))
 

{cobweb_launcher-1.2.21.dist-info → cobweb_launcher-1.2.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.21
+Version: 1.2.23
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.21.dist-info → cobweb_launcher-1.2.23.dist-info}/RECORD

@@ -14,14 +14,14 @@ cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSr
 cobweb/crawlers/crawler.py,sha256=xiFNM0t69f5xlm59hPbO2MpqtdirVAUhD84-CLpyHPM,6349
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb/db/api_db.py,sha256=646diIcZxdO0lfev958PZ-hTm5BmtJ0ns50nZOjlXOc,2865
+cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
 cobweb/db/redis_db.py,sha256=fumNZJiio-uQqRcSrymx8eJ1PqsdOwITe_Y-9JOXxrQ,4298
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
 cobweb/launchers/launcher.py,sha256=SK4f3Fpuv-QMMriHruXGQ1sh1lxT1DZ2PdG0p2wAzNw,6978
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_api.py,sha256=t-hPYhyD_Lq-75YxOXsPqTovSEuwMirCq0Tu1RKAILE,8638
+cobweb/launchers/launcher_api.py,sha256=doJ17v6DXiLB-xTfWK1giQ4MNRzuhFUpLXx7Ec7r9zg,8702
 cobweb/launchers/launcher_pro.py,sha256=2JdN4khAFGBWLXDKaWknjG72XBhfTAnwtrgVmkF0K3M,8653
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
@@ -33,8 +33,8 @@ cobweb/utils/__init__.py,sha256=vBtZTy3EfRE0MmH43URhmr7nw6_oOWTEbGOM9xR_9o8,78
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.21.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.21.dist-info/METADATA,sha256=--vQihg3yNZyTL9B02LGWLl0gKNYB0uu1ktKC_PmFXI,6510
-cobweb_launcher-1.2.21.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.21.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.21.dist-info/RECORD,,
+cobweb_launcher-1.2.23.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.23.dist-info/METADATA,sha256=qnhBqjrdIuXM7fRz3vOV_jwFLydD5eoe1QJ88bG_yMk,6510
+cobweb_launcher-1.2.23.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.23.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.23.dist-info/RECORD,,