cobweb-launcher 1.1.9__py3-none-any.whl → 1.1.10__py3-none-any.whl

cobweb/__init__.py

@@ -1,2 +1,2 @@
 from .launchers import Launcher, LauncherPro
-from .constant import Crawler
+from .constant import CrawlerModel

cobweb/constant.py

@@ -1,5 +1,5 @@
 
-class Crawler:
+class CrawlerModel:
 
     default = "cobweb.crawlers.Crawler"
     file_air = "cobweb.crawlers.FileCrawlerAir"

cobweb/crawlers/file_crawler.py

@@ -17,7 +17,7 @@ class FileCrawlerAir(Crawler):
         seed_dict = item.seed.to_dict
         bucket_name = oss_util.bucket
         try:
-            key = item.seed.oss_path
+            key = item.seed.oss_path or getattr(item, "oss_path", None)
             if oss_util.exists(key):
                 content_length = oss_util.head(key).content_length
                 yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
@@ -90,14 +90,14 @@ class FileCrawlerAir(Crawler):
             yield Seed(seed_dict, identifier="merge")
 
 
-class FileCrawlerPro(Crawler):
+class VideoCrawler(FileCrawlerAir):
 
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
         seed_dict = item.seed.to_dict
         bucket_name = oss_util.bucket
         try:
-            key = item.seed.oss_path
+            key = item.seed.oss_path or getattr(item, "oss_path", None)
             if oss_util.exists(key):
                 content_length = oss_util.head(key).content_length
                 yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)

cobweb/launchers/launcher.py

@@ -59,7 +59,7 @@ class Launcher(threading.Thread):
         _setting.update(**kwargs)
 
         for k, v in _setting.items():
-            setattr(setting, k, v)
+            setattr(setting, k.upper(), v)
 
         self._Crawler = dynamic_load_class(setting.CRAWLER)
         self._Pipeline = dynamic_load_class(setting.PIPELINE)

cobweb/setting.py

@@ -26,6 +26,7 @@ OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
 OSS_CHUNK_SIZE = 10 * 1024 ** 2
 OSS_MIN_UPLOAD_SIZE = 1024
 
+
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
 

cobweb/utils/oss.py

@@ -15,11 +15,13 @@ class OssUtil:
         secret_key=None,
         chunk_size=None,
         min_upload_size=None,
+        **kwargs
     ):
         self.bucket = bucket or setting.OSS_BUCKET
         self.endpoint = endpoint or setting.OSS_ENDPOINT
         self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
         self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
+
         self._auth = Auth(
             access_key_id=access_key or setting.OSS_ACCESS_KEY,
             access_key_secret=secret_key or setting.OSS_SECRET_KEY
@@ -28,6 +30,7 @@ class OssUtil:
             auth=self._auth,
             endpoint=self.endpoint,
             bucket_name=self.bucket,
+            **kwargs
         )
 
     def exists(self, key: str) -> bool:

{cobweb_launcher-1.1.9.dist-info → cobweb_launcher-1.1.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.9
+Version: 1.1.10
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.9.dist-info → cobweb_launcher-1.1.10.dist-info}/RECORD

@@ -1,6 +1,6 @@
-cobweb/__init__.py,sha256=UjV-xEAOr7IDUEWp8mkGvbQlOxF8JfG677Ru_OkAZZc,75
-cobweb/constant.py,sha256=-h7w2xN7WV9vrikN54TW4oSHUk0HruWpg9g2wjUoeDk,2020
-cobweb/setting.py,sha256=MRjqm75EAhqFi_6We5piF_cTCfRJojRXDKkLhP3yhvo,1951
+cobweb/__init__.py,sha256=1vB4yfJvrBVnixRRVNHLrJm8L0l5p_zOzLol5vKwP9M,80
+cobweb/constant.py,sha256=dMzvtsPje_5VgtMoOn-EvDtgycUWTKBKmDzX-CDfHDY,2025
+cobweb/setting.py,sha256=EKLY6WLIEhHSZ7Q27uccP1Knaucr72wZ_gWMD9Tieyo,1952
 cobweb/base/__init__.py,sha256=diiK5MygQaWjlWNLbW6eUIg-93O6glMGC9WLNM5jyOc,209
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -11,22 +11,22 @@ cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
 cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
 cobweb/crawlers/__init__.py,sha256=_HAXBg7Sq8fsDGSjDm3AQz9aQtLZONpt5b8dSe607mI,91
 cobweb/crawlers/base_crawler.py,sha256=uR1wQ2sJpFovNoAK52293rF03O-jNbv24P5QoNt1tW0,5169
-cobweb/crawlers/file_crawler.py,sha256=Lr0-eQJlQcgHaojOM5-awcIAJOUi9eZ_E1XOTYuQM28,8455
+cobweb/crawlers/file_crawler.py,sha256=jCdjQuNEAGHDc_y3pKFKNg8GZS_Qdxx9MywhNmLzMgA,8530
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
-cobweb/launchers/launcher.py,sha256=s4PSfcjYiMiboSB_yEOS68NSMBBE7VW5BY3WzPLxq1w,5700
+cobweb/launchers/launcher.py,sha256=XSZrrtg9ge3GLFnMY_AHuo8n1-QJSmY7j_XsheQu0rk,5708
 cobweb/launchers/launcher_pro.py,sha256=T4mgSBqFkbU64EF6Fo_oMxzWldD_-l0rC-zFEZxqBQU,6593
 cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
-cobweb/utils/oss.py,sha256=uD5aN2oVYImit3amE6TjxWMaTAcbAh9dCnpIQhf4M9Q,3238
+cobweb/utils/oss.py,sha256=qAl05ybL2Jp6KFjHDHVMfmeBHQmDKPtZleHjHAY7LZc,3277
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.9.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.9.dist-info/METADATA,sha256=BmlL5uOSiwZxO-aeZXHrmAjc_DV0rnK2HljaASlBE8w,1245
-cobweb_launcher-1.1.9.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.9.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.9.dist-info/RECORD,,
+cobweb_launcher-1.1.10.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.10.dist-info/METADATA,sha256=ZCLDIqt39zWjikRA9Z9OLNyJr55ujHOIXW9MOjZkh0g,1246
+cobweb_launcher-1.1.10.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.10.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.10.dist-info/RECORD,,