cobweb-launcher 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

cobweb/__init__.py

@@ -2,6 +2,7 @@ from .bbb import Seed, Queue, DBItem
 from .task import Task
 from .log import log
 from .db.redis_db import RedisDB
+from .db.oss_db import OssDB
 
 from .equip.distributed.launcher import launcher
 from .equip.single.launcher import launcher as single_launcher

cobweb/db/oss_db.py

@@ -13,8 +13,8 @@ class OssDB:
             endpoint,
             access_key,
             secret_key,
-            chunk_size,
-            min_size
+            chunk_size=1024 ** 2,
+            min_size=1024
     ):
         self.endpoint = endpoint
         self.bucket_name = bucket_name
@@ -27,8 +27,8 @@ class OssDB:
             endpoint=endpoint,
             bucket_name=bucket_name
         )
-        self.chunk_size = chunk_size or 1024 ** 2
-        self.min_size = min_size or 1024
+        self.chunk_size = chunk_size
+        self.min_size = min_size
 
     @staticmethod
     def format_upload_len(length):
@@ -104,14 +104,15 @@ class OssDB:
                     f", file size: {format_upload}"
                 )
 
-            else:
-                log.info(
-                    f"file size smaller than min size! "
-                    f"file size: {format_upload}"
-                )
+            # else:
+            #     log.info(
+            #         f"file size smaller than min size! "
+            #         f"file size: {format_upload}"
+            #     )
             status = True
         except ValueError as e:
-            log.exception(str(e))
+            pass
+            # log.exception(str(e))
         except oss2.exceptions.RequestError as e:
             self.bucket = oss2.Bucket(
                 auth=self.auth,

cobweb/equip/__init__.py

@@ -1,4 +1,4 @@
-from .. import Queue, DBItem, RedisDB, Seed, log
+from .. import Queue, DBItem, RedisDB, Seed, log, OssDB
 from ..constant import Setting, DealModel
 from ..utils import (
     struct_queue_name as sqn,

cobweb/equip/single/launcher.py

@@ -2,7 +2,7 @@ import time
 import threading
 
 from .. import log, sqn, rtn, pim
-from .. import Queue, DBItem, RedisDB, Setting
+from .. import Queue, DBItem, RedisDB, Setting, OssDB
 from .models import Scheduler, Spider, Storer
 
 
@@ -81,6 +81,8 @@ def launcher(task):
 
         # new item
         item = type("Item", (object,), {"redis_client": redis_db.client})()
+        if task.oss_config:
+            item.oss = OssDB(**task.oss_config)
 
         log.info("初始化cobweb!")
 

cobweb/equip/single/models.py

@@ -68,7 +68,7 @@ class Spider:
 
             try:
                 self.spider_in_progress.push(1, direct_insertion=True)
-                # log.info("spider seed: " + str(seed))
+                log.info("spider seed: " + str(seed))
                 iterators = func(item, seed)
 
                 if not isgenerator(iterators):
@@ -77,7 +77,9 @@ class Spider:
                         continue
                     raise TypeError(f"{func.__name__} isn't a generator")
 
+                status = None
                 for it in iterators:
+                    status = True
                     if getattr(it, "table_name", None):
                         store_queue = it.queue()
                         store_queue.push(
@@ -99,6 +101,10 @@ class Spider:
                     else:
                         raise TypeError("yield value type error!")
 
+                if not status:
+                    seed._retry += 1
+                    self.queue.push(seed)
+
             except Exception as e:
                 seed._retry += 1
                 self.queue.push(seed)

{cobweb_launcher-0.1.13.dist-info → cobweb_launcher-0.1.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.13
+Version: 0.1.15
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-0.1.13.dist-info → cobweb_launcher-0.1.15.dist-info}/RECORD

@@ -1,4 +1,4 @@
-cobweb/__init__.py,sha256=fIg4v2yl3NHiCSli9EHU1WnMTLqEfDAvOgRtmULwu5A,227
+cobweb/__init__.py,sha256=rd5vkgxe49DzVH4oyvwl-pWQeJrqF1K6XT1NZfvMmTo,256
 cobweb/bbb.py,sha256=zKeCeBVFQfhEv6M8CCktUTM7tXDZmAu6ZN0-ET44pUY,5707
 cobweb/constant.py,sha256=t0_2xY_VU6T18DnD8tL9LDQJB6ibE9UWx66H5yz_934,489
 cobweb/decorators.py,sha256=eYQI9rddPVJihAlomLTmbtQhIOzPw8dCrOFpxAq2pLY,318
@@ -8,7 +8,7 @@ cobweb/setting.py,sha256=UAu_dLuIFYO98MxtlZ5sZqJcwKAUKq4Bu4KoKlV50Mc,288
 cobweb/task.py,sha256=okE6rBFm9VnYOAXShuTXJ0uvc-_ITsuuxhyxNKbymmU,1775
 cobweb/utils.py,sha256=ivmRqJJNtwdOKYT4G7qQCWnL8ar9c-shxeDZzGB2E9c,2651
 cobweb/db/__init__.py,sha256=jC-uOThYLtiDUG6cTJRkDITgOzR4nIOeaZQeZhuk-v0,139
-cobweb/db/oss_db.py,sha256=lFGNuH3tdIMsohVXQ_fTZPyBfS2oxYNmFNuQ-ZBQgm0,4221
+cobweb/db/oss_db.py,sha256=59HwMMfoGUr6l_BI4p84YHYoQwEn22cCM_1GZK34Uo4,4244
 cobweb/db/redis_db.py,sha256=go9IPQQZAl_jXiHDTruWF7N8svs9QEpQPOr7JhuLCQs,8179
 cobweb/db/scheduler/__init__.py,sha256=w5uIGEB1wLJ-H9RqGpzRwOEWW-BBVSk6Cc7FxZIlWCs,51
 cobweb/db/scheduler/default.py,sha256=XDtxNyu5KTpVAbfCOW8mR1zNFNHiMuaQ4sAhZuIYBoM,79
@@ -21,19 +21,19 @@ cobweb/db/storer/textfile.py,sha256=auoXGXLbIbEhMoeYIhy58qw22N2r0fQTtzVjHCjqVGA,
 cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/distributed/launcher.py,sha256=jTtBXBmna_6yFdj6gyGQiiEtg8I0g5uI5h8kbHWt454,7998
 cobweb/distributed/models.py,sha256=PUQokXMGD-H4A99nX7qYA395Ul6IsWGruMTVa05nswY,4568
-cobweb/equip/__init__.py,sha256=UVhm9xl9kj1Ez_9Sf5ElRvkeI5pTpXatWfIHJbXAFx4,240
+cobweb/equip/__init__.py,sha256=LWhbrTnG9kD1et0D40EzLISPuE0PIS-5WD3y3CLDaWk,247
 cobweb/equip/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/equip/distributed/launcher.py,sha256=1LzxibGXWR20XpXawakiRpEMaa9yfaj2rFSKnmEwjFc,7475
 cobweb/equip/distributed/models.py,sha256=qTGzxLdb2arsZSZK2HE4-MrqhraUhc2Ol5wBvlv_aWA,5008
 cobweb/equip/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/equip/single/launcher.py,sha256=dMd1IJg__yiFnxy3Q6EueebPllN8zyTDwfC0OjEhRf4,6702
-cobweb/equip/single/models.py,sha256=RFhodfO9UDba2C4a3yHVrbzv-CSqfgbR2zAH8U-O7gk,4640
+cobweb/equip/single/launcher.py,sha256=KRsw7yxklvFM85cel-EyLsNPLyrC9Hd26BMSx6-4Hac,6785
+cobweb/equip/single/models.py,sha256=w3LQEhTrgqoYZn5v9TiEK2A68xuC7QH7suRP9OYnoOg,4813
 cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/single/launcher.py,sha256=IoJbn87j7t7Pib_FxoWZmmX8asXOqNGb-9ospw6EYJI,7302
 cobweb/single/models.py,sha256=UXcxr_Quok91k82plaqbj4deB-UBCWo14WCo6SS5L_o,4247
 cobweb/single/nest.py,sha256=49K6KQ934INfPrWQsrq9rIFpQauLbLGOFbDaHvoQzOk,5015
-cobweb_launcher-0.1.13.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.1.13.dist-info/METADATA,sha256=ffqwHcTVid1e3EhNB6WpThreoca8UabGjTxDHKE3CME,1220
-cobweb_launcher-0.1.13.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cobweb_launcher-0.1.13.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.1.13.dist-info/RECORD,,
+cobweb_launcher-0.1.15.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.1.15.dist-info/METADATA,sha256=xkxL2woC_kK4XxUfrCaf80SeA5SE9t_0vwMynXt90lc,1220
+cobweb_launcher-0.1.15.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+cobweb_launcher-0.1.15.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.1.15.dist-info/RECORD,,