cobweb-launcher 1.0.9__tar.gz → 1.1.1__tar.gz

{cobweb-launcher-1.0.9/cobweb_launcher.egg-info → cobweb-launcher-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.0.9
+Version: 1.1.1
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.1}/cobweb/crawlers/base_crawler.py

@@ -37,7 +37,7 @@ class Crawler(threading.Thread):
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
         response = item.download()
-        yield Response(item.seed, response)
+        yield Response(item.seed, response, **item.to_dict)
 
     @staticmethod
     def parse(item: Response) -> BaseItem:
@@ -54,6 +54,7 @@ class Crawler(threading.Thread):
                 continue
 
             elif seed.params.retry >= self.max_retries:
+                seed.params.identifier = DealModel.fail
                 self.launcher_queue['done'].push(seed)
                 continue
 

{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.1}/cobweb/db/redis_db.py

@@ -126,3 +126,4 @@ class RedisDB:
         self.execute_lua(lua_script, keys, *args)
 
 
+

{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.1}/cobweb/launchers/launcher_pro.py

@@ -16,7 +16,6 @@ class LauncherPro(Launcher):
         self._fail = "{%s:%s}:fail" % (project, task)
         self._heartbeat = "heartbeat:%s_%s" % (project, task)
         self._reset_lock = "lock:reset:%s_%s" % (project, task)
-        self._heartbeat_lock = "lock:heartbeat:%s_%s" % (project, task)
         self._db = RedisDB()
 
         self._heartbeat_start_event = threading.Event()
@@ -29,10 +28,8 @@ class LauncherPro(Launcher):
     def _execute_heartbeat(self):
         while not self._stop.is_set():
             if self._heartbeat_start_event.is_set():
-                if self._db.lock(self._heartbeat_lock, t=1):
-                    self._db.setex(self._heartbeat, 3)
-                time.sleep(1)
-            time.sleep(0.5)
+                self._db.setex(self._heartbeat, 3)
+            time.sleep(2)
 
     def _reset(self):
         """

{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.1}/cobweb/setting.py

@@ -23,8 +23,8 @@ OSS_BUCKET = os.getenv("OSS_BUCKET")
 OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
 OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
 OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
+OSS_CHUNK_SIZE = 10 * 1024 ** 2
 OSS_MIN_UPLOAD_SIZE = 1024
-OSS_CHUNK_SIZE = 1024 ** 2
 
 # 采集器选择
 CRAWLER = "cobweb.crawlers.CrawlerAir"

{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.1/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.0.9
+Version: 1.1.1
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.0.9 → cobweb-launcher-1.1.1}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.0.9",
+    version="1.1.1",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",