cobweb-launcher 1.2.63__tar.gz → 1.2.65__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {cobweb-launcher-1.2.63/cobweb_launcher.egg-info → cobweb-launcher-1.2.65}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/crawlers/crawler.py +2 -0
  3. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/launchers/launcher_api.py +2 -2
  4. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/launchers/launcher_pro.py +2 -2
  5. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  6. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/setup.py +1 -1
  7. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/LICENSE +0 -0
  8. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/README.md +0 -0
  9. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/__init__.py +0 -0
  10. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/base/__init__.py +0 -0
  11. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/base/common_queue.py +0 -0
  12. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/base/decorators.py +0 -0
  13. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/base/item.py +0 -0
  14. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/base/log.py +0 -0
  15. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/base/request.py +0 -0
  16. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/base/response.py +0 -0
  17. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/base/seed.py +0 -0
  18. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/constant.py +0 -0
  19. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/crawlers/__init__.py +0 -0
  20. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/crawlers/base_crawler.py +0 -0
  21. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/crawlers/file_crawler.py +0 -0
  22. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/db/__init__.py +0 -0
  23. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/db/api_db.py +0 -0
  24. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/db/redis_db.py +0 -0
  25. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/db/redis_db_new.py +0 -0
  26. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/exceptions/__init__.py +0 -0
  27. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/exceptions/oss_db_exception.py +0 -0
  28. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/launchers/__init__.py +0 -0
  29. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/launchers/launcher.py +0 -0
  30. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/launchers/launcher_air.py +0 -0
  31. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/pipelines/__init__.py +0 -0
  32. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/pipelines/pipeline.py +0 -0
  33. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/pipelines/pipeline_console.py +0 -0
  34. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/pipelines/pipeline_loghub.py +0 -0
  35. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/setting.py +0 -0
  36. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/utils/__init__.py +0 -0
  37. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/utils/bloom.py +0 -0
  38. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/utils/dotting.py +0 -0
  39. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/utils/oss.py +0 -0
  40. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/utils/tools.py +0 -0
  41. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  42. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  43. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb_launcher.egg-info/requires.txt +0 -0
  44. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb_launcher.egg-info/top_level.txt +0 -0
  45. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/setup.cfg +0 -0
  46. {cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/test/test.py +0 -0
{cobweb-launcher-1.2.63/cobweb_launcher.egg-info → cobweb-launcher-1.2.65}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.63
3
+ Version: 1.2.65
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/crawlers/crawler.py RENAMED
@@ -149,6 +149,7 @@ class Crawler(threading.Thread):
149
149
  data_size=int(download_item.response.headers.get("content-length", 0)),
150
150
  cost_time=end_time - start_time, status = 200,
151
151
  url=download_item.response.url,
152
+ seed=download_item.seed.to_string,
152
153
  proxy_type=seed.params.proxy_type,
153
154
  proxy=seed.params.proxy,
154
155
  project=self.project, task=self.task,
@@ -179,6 +180,7 @@ class Crawler(threading.Thread):
179
180
  topic=urlparse(url).netloc,
180
181
  data_size=-1, cost_time=-1,
181
182
  status=status, url=url,
183
+ seed=seed.to_string,
182
184
  proxy_type=seed.params.proxy_type,
183
185
  proxy=seed.params.proxy,
184
186
  project=self.project,
{cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/launchers/launcher_api.py RENAMED
@@ -118,11 +118,11 @@ class LauncherApi(Launcher):
118
118
  break
119
119
  seed, new_seed = seed_tuple
120
120
  new_seeds[new_seed.to_string] = new_seed.params.priority
121
- del_seeds.add(seed.to_string)
121
+ del_seeds.add(seed)
122
122
  if new_seeds:
123
123
  self._db.zadd(self._todo_key, new_seeds, nx=True)
124
124
  if del_seeds:
125
- self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
125
+ self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
126
126
  if status:
127
127
  time.sleep(self._new_queue_wait_seconds)
128
128
 
{cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/cobweb/launchers/launcher_pro.py RENAMED
@@ -117,11 +117,11 @@ class LauncherPro(Launcher):
117
117
  break
118
118
  seed, new_seed = seed_tuple
119
119
  new_seeds[new_seed.to_string] = new_seed.params.priority
120
- del_seeds.add(seed.to_string)
120
+ del_seeds.add(seed)
121
121
  if new_seeds:
122
122
  self._db.zadd(self._todo_key, new_seeds, nx=True)
123
123
  if del_seeds:
124
- self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
124
+ self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
125
125
  if status:
126
126
  time.sleep(self._new_queue_wait_seconds)
127
127
 
{cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.63
3
+ Version: 1.2.65
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.63 → cobweb-launcher-1.2.65}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.2.63",
8
+ version="1.2.65",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",