cobweb-launcher 1.2.60__py3-none-any.whl → 1.2.62__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -83,7 +83,7 @@ class Crawler(threading.Thread):
         if isinstance(item, BaseItem):
             self._upload_data(item)
         elif isinstance(item, Seed):
-            self._add_seed(item)
+            self._add_seed((seed, item))
         elif isinstance(item, str) and item == DealModel.poll:
             self._set_seed(seed)
         elif isinstance(item, str) and item == DealModel.done:

cobweb/launchers/launcher.py

@@ -173,8 +173,8 @@ class Launcher(threading.Thread):
     def _upload_data(self, data, **kwargs):
         self.__LAUNCHER_QUEUE__["upload"].push(data, **kwargs)
 
-    def _add_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["new"].push(seed, **kwargs)
+    def _add_seed(self, seeds, **kwargs):
+        self.__LAUNCHER_QUEUE__["new"].push(seeds, direct_insertion=True, **kwargs)
 
     def _delete_seed(self, seed, **kwargs):
         self.__LAUNCHER_QUEUE__["done"].push(seed, **kwargs)

cobweb/launchers/launcher_air.py

@@ -13,15 +13,20 @@ class LauncherAir(Launcher):
 
     @check_pause
     def _insert(self):
-        seeds = {}
+        new_seeds = {}
+        del_seeds = set()
         status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
         for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed:
+            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed_tuple:
                 break
-            seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(seeds)
+            seed, new_seed = seed_tuple
+            new_seeds[new_seed.to_string] = new_seed.params.priority
+            del_seeds.add(seed.to_string)
+        if new_seeds:
+            self.__LAUNCHER_QUEUE__['todo'].push(new_seeds)
+        if del_seeds:
+            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

cobweb/launchers/launcher_api.py

@@ -106,14 +106,20 @@ class LauncherApi(Launcher):
         """
         添加新种子到redis队列中
         """
-        seeds = {}
+        new_seeds = {}
+        del_seeds = set()
         status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
         for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if seed:
-                seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self._db.zadd(self._todo_key, seeds, nx=True)
+            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed_tuple:
+                break
+            seed, new_seed = seed_tuple
+            new_seeds[new_seed.to_string] = new_seed.params.priority
+            del_seeds.add(seed.to_string)
+        if new_seeds:
+            self._db.zadd(self._todo_key, new_seeds, nx=True)
+        if del_seeds:
+            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

cobweb/launchers/launcher_pro.py

@@ -108,14 +108,20 @@ class LauncherPro(Launcher):
         """
         添加新种子到redis队列中
         """
-        seeds = {}
+        new_seeds = {}
+        del_seeds = set()
         status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
         for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if seed:
-                seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self._db.zadd(self._todo_key, seeds, nx=True)
+            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed_tuple:
+                break
+            seed, new_seed = seed_tuple
+            new_seeds[new_seed.to_string] = new_seed.params.priority
+            del_seeds.add(seed.to_string)
+        if new_seeds:
+            self._db.zadd(self._todo_key, new_seeds, nx=True)
+        if del_seeds:
+            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

cobweb/utils/oss.py

@@ -1,4 +1,4 @@
-from typing import List
+
 from cobweb import setting
 from requests import Response
 from oss2 import Auth, Bucket, models, PartIterator
@@ -23,6 +23,9 @@ class OssUtil:
         self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
         self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
 
+        self.failed_count = 0
+        self._kw = kwargs
+
         self._auth = Auth(
             access_key_id=access_key or setting.OSS_ACCESS_KEY,
             access_key_secret=secret_key or setting.OSS_SECRET_KEY
@@ -31,11 +34,27 @@ class OssUtil:
             auth=self._auth,
             endpoint=self.endpoint,
             bucket_name=self.bucket,
-            **kwargs
+            **self._kw
         )
 
+    def failed(self):
+        self.failed_count += 1
+        if self.failed_count >= 5:
+            self._client = Bucket(
+                auth=self._auth,
+                endpoint=self.endpoint,
+                bucket_name=self.bucket,
+                **self._kw
+            )
+
     def exists(self, key: str) -> bool:
-        return self._client.object_exists(key)
+        try:
+            result = self._client.object_exists(key)
+            self.failed_count = 0
+            return result
+        except Exception as e:
+            self.failed()
+            raise e
 
     def head(self, key: str) -> models.HeadObjectResult:
         return self._client.head_object(key)

{cobweb_launcher-1.2.60.dist-info → cobweb_launcher-1.2.62.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.60
+Version: 1.2.62
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.60.dist-info → cobweb_launcher-1.2.62.dist-info}/RECORD

@@ -13,7 +13,7 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=A-F1urjbE5hYNWTCwq3sUV4nrxlK_RGMoCmjBmIwYsI,3158
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=pePDGroD6JJAht5QTU51L7MkFIY4ob9TKpznZ2wUmsw,8965
+cobweb/crawlers/crawler.py,sha256=kbpgBllmA2ve3Hp-XvVH89t2q5G39i-m7it5xJ1p1WE,8973
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -22,10 +22,10 @@ cobweb/db/redis_db_new.py,sha256=F09LWVjtC2JFdCaKatZ2bAOLKbsnes85_nZRe2dtSIc,469
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
-cobweb/launchers/launcher.py,sha256=y1a72sANV-oJVpy_BI7BwKy7fY4AnpHqJK3t6czoCts,7896
-cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_api.py,sha256=ydx_xrOjGMhGTDdyN3Oxy5EHUvvbvetNj3mCX1GlNsw,8205
-cobweb/launchers/launcher_pro.py,sha256=0Z3E1HA_D-yy_7hD7Q-HkYdy4v4AHYwyrR4i6HErIBo,8147
+cobweb/launchers/launcher.py,sha256=aR1cnQymjQQUJe-W_dRoQTd9t3Qav9QkLDzeWW7i0xE,7921
+cobweb/launchers/launcher_air.py,sha256=qdcxq41I9zN5snEpMUUNEQNTtUiZM1Hw_3N9zu4PuAs,3058
+cobweb/launchers/launcher_api.py,sha256=RlWuGSb_izevqJBVlyi_D8mlUXMKReTSOHTHHMEn8FE,8453
+cobweb/launchers/launcher_pro.py,sha256=Jn5T89eWZafr4wGTllqrXB5IDyZ8G9hiuk7dW3Xl1VA,8395
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -36,10 +36,10 @@ cobweb/schedulers/scheduler_redis.py,sha256=E5fjc3nNld8GbUhUGT7uY4smRejj2J2ZIzp2
 cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
-cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
+cobweb/utils/oss.py,sha256=6x_ugXanh1R-6ZylQzUDQh4OeFZHujhWFCOxbzy53JY,3984
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.60.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.60.dist-info/METADATA,sha256=9RTAhglArwI9C7Vr_oIxpE-2i7V3gg15RucZJ3r58n4,6510
-cobweb_launcher-1.2.60.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.60.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.60.dist-info/RECORD,,
+cobweb_launcher-1.2.62.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.62.dist-info/METADATA,sha256=fgQXgBFjxcRQdCLc4m_2qmM11V8QHleCbROVG2dPIFo,6510
+cobweb_launcher-1.2.62.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.62.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.62.dist-info/RECORD,,