cobweb-launcher 1.1.6__tar.gz → 1.1.10__tar.gz

{cobweb-launcher-1.1.6/cobweb_launcher.egg-info → cobweb-launcher-1.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.6
+Version: 1.1.10
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.6 → cobweb-launcher-1.1.10}/cobweb/__init__.py

@@ -1,2 +1,2 @@
 from .launchers import Launcher, LauncherPro
-from .constant import Crawler
+from .constant import CrawlerModel

{cobweb-launcher-1.1.6 → cobweb-launcher-1.1.10}/cobweb/constant.py

@@ -1,5 +1,5 @@
 
-class Crawler:
+class CrawlerModel:
 
     default = "cobweb.crawlers.Crawler"
     file_air = "cobweb.crawlers.FileCrawlerAir"

{cobweb-launcher-1.1.6 → cobweb-launcher-1.1.10}/cobweb/crawlers/file_crawler.py

@@ -17,7 +17,7 @@ class FileCrawlerAir(Crawler):
         seed_dict = item.seed.to_dict
         bucket_name = oss_util.bucket
         try:
-            key = item.seed.oss_path
+            key = item.seed.oss_path or getattr(item, "oss_path", None)
             if oss_util.exists(key):
                 content_length = oss_util.head(key).content_length
                 yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
@@ -90,14 +90,14 @@ class FileCrawlerAir(Crawler):
             yield Seed(seed_dict, identifier="merge")
 
 
-class FileCrawlerPro(Crawler):
+class VideoCrawler(FileCrawlerAir):
 
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
         seed_dict = item.seed.to_dict
         bucket_name = oss_util.bucket
         try:
-            key = item.seed.oss_path
+            key = item.seed.oss_path or getattr(item, "oss_path", None)
             if oss_util.exists(key):
                 content_length = oss_util.head(key).content_length
                 yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
@@ -123,12 +123,14 @@ class FileCrawlerPro(Crawler):
                         item.seed, response, filter=True, msg=f"response content type is {content_type}",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
+                    response.close()
                 elif position == 1 and min_upload_size >= content_length > 0:
                     """过小文件标识返回"""
                     yield Response(
                         item.seed, response, filter=True, msg="file size is too small",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
+                    response.close()
                 elif position == 1 and chunk_size > content_length > min_upload_size:
                     """小文件直接下载"""
                     for part_data in response.iter_content(chunk_size):
@@ -153,12 +155,12 @@ class FileCrawlerPro(Crawler):
                             seed_dict['position'] = position
                             seed_dict['start'] = upload_content_length
 
-                    response.close()
                     if content:
                         oss_util.put_part(key, upload_id, position, content)
                         content_length += len(content)
                     oss_util.merge(key, upload_id)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                    response.close()
 
             elif item.seed.params.identifier == "merge":
                 oss_util.merge(key, seed_dict["upload_id"])

{cobweb-launcher-1.1.6 → cobweb-launcher-1.1.10}/cobweb/launchers/launcher.py

@@ -34,7 +34,7 @@ class Launcher(threading.Thread):
         "_delete",
     ]
 
-    def __init__(self, task, project, custom_setting=None):
+    def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__()
         self.task = task
         self.project = project
@@ -42,10 +42,11 @@ class Launcher(threading.Thread):
         self._stop = threading.Event()  # 结束事件
         self._pause = threading.Event()  # 暂停事件
 
+        _setting = dict()
+
         if custom_setting:
-            setting_ = dict()
             if isinstance(custom_setting, dict):
-                setting_ = custom_setting
+                _setting = custom_setting
             else:
                 if isinstance(custom_setting, str):
                     custom_setting = importlib.import_module(custom_setting)
@@ -53,9 +54,12 @@ class Launcher(threading.Thread):
                     raise Exception
                 for k, v in custom_setting.__dict__.items():
                     if not k.startswith("__") and not inspect.ismodule(v):
-                        setting_[k] = v
-            for k, v in setting_.items():
-                setattr(setting, k, v)
+                        _setting[k] = v
+
+        _setting.update(**kwargs)
+
+        for k, v in _setting.items():
+            setattr(setting, k.upper(), v)
 
         self._Crawler = dynamic_load_class(setting.CRAWLER)
         self._Pipeline = dynamic_load_class(setting.PIPELINE)

{cobweb-launcher-1.1.6 → cobweb-launcher-1.1.10}/cobweb/launchers/launcher_pro.py

@@ -9,8 +9,8 @@ from cobweb.constant import DealModel, LogTemplate
 
 class LauncherPro(Launcher):
 
-    def __init__(self, task, project, custom_setting=None):
-        super().__init__(task, project, custom_setting)
+    def __init__(self, task, project, custom_setting=None, **kwargs):
+        super().__init__(task, project, custom_setting, **kwargs)
         self._todo = "{%s:%s}:todo" % (project, task)
         self._done = "{%s:%s}:done" % (project, task)
         self._fail = "{%s:%s}:fail" % (project, task)
@@ -147,12 +147,12 @@ class LauncherPro(Launcher):
                 self._pause.clear()
                 self._execute()
             elif queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
+                pooling_wait_seconds = 5
                 check_emtpy_times += 1
             else:
                 check_emtpy_times = 0
 
-            if not self._db.zcard(self._todo) and check_emtpy_times > 2:
+            if not self._db.zcount(self._todo, _min=0, _max="(1000") and check_emtpy_times > 2:
                 check_emtpy_times = 0
                 self.__DOING__ = {}
                 self._pause.set()

{cobweb-launcher-1.1.6 → cobweb-launcher-1.1.10}/cobweb/setting.py

@@ -26,6 +26,7 @@ OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
 OSS_CHUNK_SIZE = 10 * 1024 ** 2
 OSS_MIN_UPLOAD_SIZE = 1024
 
+
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
 

{cobweb-launcher-1.1.6 → cobweb-launcher-1.1.10}/cobweb/utils/oss.py

@@ -15,11 +15,13 @@ class OssUtil:
         secret_key=None,
         chunk_size=None,
         min_upload_size=None,
+        **kwargs
     ):
         self.bucket = bucket or setting.OSS_BUCKET
         self.endpoint = endpoint or setting.OSS_ENDPOINT
         self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
         self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
+
         self._auth = Auth(
             access_key_id=access_key or setting.OSS_ACCESS_KEY,
             access_key_secret=secret_key or setting.OSS_SECRET_KEY
@@ -28,6 +30,7 @@ class OssUtil:
             auth=self._auth,
             endpoint=self.endpoint,
             bucket_name=self.bucket,
+            **kwargs
         )
 
     def exists(self, key: str) -> bool:

{cobweb-launcher-1.1.6 → cobweb-launcher-1.1.10/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.6
+Version: 1.1.10
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.6 → cobweb-launcher-1.1.10}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.1.6",
+    version="1.1.10",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",
@@ -22,4 +22,4 @@ setup(
         "cobweb-launcher, cobweb",
     ],
     python_requires=">=3.7",
-)
+)