cobweb-launcher 1.2.47__tar.gz → 1.2.49__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (45) hide show
  1. {cobweb-launcher-1.2.47/cobweb_launcher.egg-info → cobweb-launcher-1.2.49}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/crawlers/crawler.py +3 -0
  3. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/utils/dotting.py +3 -2
  4. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  5. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/setup.py +1 -1
  6. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/LICENSE +0 -0
  7. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/README.md +0 -0
  8. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/__init__.py +0 -0
  9. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/base/__init__.py +0 -0
  10. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/base/common_queue.py +0 -0
  11. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/base/decorators.py +0 -0
  12. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/base/item.py +0 -0
  13. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/base/log.py +0 -0
  14. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/base/request.py +0 -0
  15. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/base/response.py +0 -0
  16. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/base/seed.py +0 -0
  17. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/constant.py +0 -0
  18. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/crawlers/__init__.py +0 -0
  19. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/crawlers/base_crawler.py +0 -0
  20. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/crawlers/file_crawler.py +0 -0
  21. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/db/__init__.py +0 -0
  22. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/db/api_db.py +0 -0
  23. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/db/redis_db.py +0 -0
  24. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/exceptions/__init__.py +0 -0
  25. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/exceptions/oss_db_exception.py +0 -0
  26. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/launchers/__init__.py +0 -0
  27. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/launchers/launcher.py +0 -0
  28. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/launchers/launcher_air.py +0 -0
  29. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/launchers/launcher_api.py +0 -0
  30. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/launchers/launcher_pro.py +0 -0
  31. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/pipelines/__init__.py +0 -0
  32. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/pipelines/pipeline.py +0 -0
  33. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/pipelines/pipeline_console.py +0 -0
  34. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/pipelines/pipeline_loghub.py +0 -0
  35. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/setting.py +0 -0
  36. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/utils/__init__.py +0 -0
  37. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/utils/bloom.py +0 -0
  38. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/utils/oss.py +0 -0
  39. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/utils/tools.py +0 -0
  40. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  41. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  42. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb_launcher.egg-info/requires.txt +0 -0
  43. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb_launcher.egg-info/top_level.txt +0 -0
  44. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/setup.cfg +0 -0
  45. {cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/test/test.py +0 -0
{cobweb-launcher-1.2.47/cobweb_launcher.egg-info → cobweb-launcher-1.2.49}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.47
3
+ Version: 1.2.49
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/crawlers/crawler.py RENAMED
@@ -150,6 +150,7 @@ class Crawler(threading.Thread):
150
150
  data_size=int(download_item.response.headers.get("content-length", 0)),
151
151
  cost_time=end_time - start_time, status = 200,
152
152
  url=download_item.response.url, proxy_type=proxy_type,
153
+ project=self.project, task=self.task,
153
154
  )
154
155
  parse_iterators = self.parse(download_item)
155
156
  if not isgenerator(parse_iterators):
@@ -178,6 +179,8 @@ class Crawler(threading.Thread):
178
179
  data_size=-1, cost_time=-1,
179
180
  status=status, url=url,
180
181
  proxy_type=proxy_type,
182
+ project=self.project,
183
+ task=self.task,
181
184
  msg=exception_msg,
182
185
  )
183
186
  logger.info(LogTemplate.download_exception.format(
{cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/cobweb/utils/dotting.py RENAMED
@@ -46,9 +46,10 @@ class LoghubDot:
46
46
  try:
47
47
  log_item_info = {}
48
48
  for _ in range(1000):
49
- topic, item = self.queue.pop()
50
- if not item:
49
+ its = self.queue.pop()
50
+ if not its:
51
51
  break
52
+ topic, item = its
52
53
  log_item_info.setdefault(topic, []).append(item)
53
54
  for topic, log_items in log_item_info.items():
54
55
  request = PutLogsRequest(
{cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.47
3
+ Version: 1.2.49
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.47 → cobweb-launcher-1.2.49}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.2.47",
8
+ version="1.2.49",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",