cobweb-launcher 1.2.60__tar.gz → 1.2.61__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {cobweb-launcher-1.2.60/cobweb_launcher.egg-info → cobweb-launcher-1.2.61}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/utils/oss.py +22 -3
  3. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  4. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/setup.py +1 -1
  5. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/LICENSE +0 -0
  6. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/README.md +0 -0
  7. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/__init__.py +0 -0
  8. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/base/__init__.py +0 -0
  9. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/base/common_queue.py +0 -0
  10. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/base/decorators.py +0 -0
  11. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/base/item.py +0 -0
  12. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/base/log.py +0 -0
  13. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/base/request.py +0 -0
  14. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/base/response.py +0 -0
  15. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/base/seed.py +0 -0
  16. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/constant.py +0 -0
  17. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/crawlers/__init__.py +0 -0
  18. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/crawlers/base_crawler.py +0 -0
  19. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/crawlers/crawler.py +0 -0
  20. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/crawlers/file_crawler.py +0 -0
  21. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/db/__init__.py +0 -0
  22. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/db/api_db.py +0 -0
  23. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/db/redis_db.py +0 -0
  24. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/db/redis_db_new.py +0 -0
  25. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/exceptions/__init__.py +0 -0
  26. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/exceptions/oss_db_exception.py +0 -0
  27. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/launchers/__init__.py +0 -0
  28. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/launchers/launcher.py +0 -0
  29. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/launchers/launcher_air.py +0 -0
  30. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/launchers/launcher_api.py +0 -0
  31. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/launchers/launcher_pro.py +0 -0
  32. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/pipelines/__init__.py +0 -0
  33. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/pipelines/pipeline.py +0 -0
  34. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/pipelines/pipeline_console.py +0 -0
  35. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/pipelines/pipeline_loghub.py +0 -0
  36. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/setting.py +0 -0
  37. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/utils/__init__.py +0 -0
  38. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/utils/bloom.py +0 -0
  39. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/utils/dotting.py +0 -0
  40. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/utils/tools.py +0 -0
  41. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  42. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  43. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb_launcher.egg-info/requires.txt +0 -0
  44. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb_launcher.egg-info/top_level.txt +0 -0
  45. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/setup.cfg +0 -0
  46. {cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/test/test.py +0 -0
{cobweb-launcher-1.2.60/cobweb_launcher.egg-info → cobweb-launcher-1.2.61}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.60
3
+ Version: 1.2.61
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/cobweb/utils/oss.py RENAMED
@@ -1,4 +1,4 @@
1
- from typing import List
1
+
2
2
  from cobweb import setting
3
3
  from requests import Response
4
4
  from oss2 import Auth, Bucket, models, PartIterator
@@ -23,6 +23,9 @@ class OssUtil:
23
23
  self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
24
24
  self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
25
25
 
26
+ self.failed_count = 0
27
+ self._kw = kwargs
28
+
26
29
  self._auth = Auth(
27
30
  access_key_id=access_key or setting.OSS_ACCESS_KEY,
28
31
  access_key_secret=secret_key or setting.OSS_SECRET_KEY
@@ -31,11 +34,27 @@ class OssUtil:
31
34
  auth=self._auth,
32
35
  endpoint=self.endpoint,
33
36
  bucket_name=self.bucket,
34
- **kwargs
37
+ **self._kw
35
38
  )
36
39
 
40
+ def failed(self):
41
+ self.failed_count += 1
42
+ if self.failed_count >= 5:
43
+ self._client = Bucket(
44
+ auth=self._auth,
45
+ endpoint=self.endpoint,
46
+ bucket_name=self.bucket,
47
+ **self._kw
48
+ )
49
+
37
50
  def exists(self, key: str) -> bool:
38
- return self._client.object_exists(key)
51
+ try:
52
+ result = self._client.object_exists(key)
53
+ self.failed_count = 0
54
+ return result
55
+ except Exception as e:
56
+ self.failed()
57
+ raise e
39
58
 
40
59
  def head(self, key: str) -> models.HeadObjectResult:
41
60
  return self._client.head_object(key)
{cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.60
3
+ Version: 1.2.61
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.60 → cobweb-launcher-1.2.61}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.2.60",
8
+ version="1.2.61",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",