cobweb-launcher 1.2.63__py3-none-any.whl → 1.2.65__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -149,6 +149,7 @@ class Crawler(threading.Thread):
                                         data_size=int(download_item.response.headers.get("content-length", 0)),
                                         cost_time=end_time - start_time, status = 200,
                                         url=download_item.response.url,
+                                        seed=download_item.seed.to_string,
                                         proxy_type=seed.params.proxy_type,
                                         proxy=seed.params.proxy,
                                         project=self.project, task=self.task,
@@ -179,6 +180,7 @@ class Crawler(threading.Thread):
                     topic=urlparse(url).netloc,
                     data_size=-1, cost_time=-1,
                     status=status, url=url,
+                    seed=seed.to_string,
                     proxy_type=seed.params.proxy_type,
                     proxy=seed.params.proxy,
                     project=self.project,

cobweb/launchers/launcher_api.py

@@ -118,11 +118,11 @@ class LauncherApi(Launcher):
                 break
             seed, new_seed = seed_tuple
             new_seeds[new_seed.to_string] = new_seed.params.priority
-            del_seeds.add(seed.to_string)
+            del_seeds.add(seed)
         if new_seeds:
             self._db.zadd(self._todo_key, new_seeds, nx=True)
         if del_seeds:
-            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
+            self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

cobweb/launchers/launcher_pro.py

@@ -117,11 +117,11 @@ class LauncherPro(Launcher):
                 break
             seed, new_seed = seed_tuple
             new_seeds[new_seed.to_string] = new_seed.params.priority
-            del_seeds.add(seed.to_string)
+            del_seeds.add(seed)
         if new_seeds:
             self._db.zadd(self._todo_key, new_seeds, nx=True)
         if del_seeds:
-            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
+            self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

{cobweb_launcher-1.2.63.dist-info → cobweb_launcher-1.2.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.63
+Version: 1.2.65
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.63.dist-info → cobweb_launcher-1.2.65.dist-info}/RECORD

@@ -13,7 +13,7 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=A-F1urjbE5hYNWTCwq3sUV4nrxlK_RGMoCmjBmIwYsI,3158
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=kbpgBllmA2ve3Hp-XvVH89t2q5G39i-m7it5xJ1p1WE,8973
+cobweb/crawlers/crawler.py,sha256=yV0R-AoVb7xg722zpIEs9aYFYV2MfZ634qzKJgDXbZo,9089
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -24,8 +24,8 @@ cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BX
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
 cobweb/launchers/launcher.py,sha256=aR1cnQymjQQUJe-W_dRoQTd9t3Qav9QkLDzeWW7i0xE,7921
 cobweb/launchers/launcher_air.py,sha256=qdcxq41I9zN5snEpMUUNEQNTtUiZM1Hw_3N9zu4PuAs,3058
-cobweb/launchers/launcher_api.py,sha256=bllvBDi2FqPtdcR2mhcdRr7CYwO5GiNVlo1p0gOCHhA,8632
-cobweb/launchers/launcher_pro.py,sha256=o2selUZaHBh_kIs1MUgu-EuueWARV8HqQUg-VoVN5yg,8428
+cobweb/launchers/launcher_api.py,sha256=3-A6k3Igvi-xnvP9M_3NkJKUweDJ_pY10ZrHClteD-g,8628
+cobweb/launchers/launcher_pro.py,sha256=xM2z3nCTb6gehuD1Ggyvli2knGAPjGczbjOhKWswVpo,8424
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -38,8 +38,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=6x_ugXanh1R-6ZylQzUDQh4OeFZHujhWFCOxbzy53JY,3984
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.63.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.63.dist-info/METADATA,sha256=N2ZvAiGMrpAXrCqpjt0IJce5atZzTyCTJkDLoTMzLvQ,6510
-cobweb_launcher-1.2.63.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.63.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.63.dist-info/RECORD,,
+cobweb_launcher-1.2.65.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.65.dist-info/METADATA,sha256=zFRzApm4_ADTxcg4-18wMQRM6AcxeD0Oww0lJY7_DBk,6510
+cobweb_launcher-1.2.65.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.65.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.65.dist-info/RECORD,,