cobweb-launcher 1.2.58__tar.gz → 1.2.59__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (46) hide show
  1. {cobweb-launcher-1.2.58/cobweb_launcher.egg-info → cobweb-launcher-1.2.59}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/db/redis_db.py +67 -2
  3. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/launcher_pro.py +0 -1
  4. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  5. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/setup.py +1 -1
  6. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/LICENSE +0 -0
  7. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/README.md +0 -0
  8. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/__init__.py +0 -0
  9. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/__init__.py +0 -0
  10. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/common_queue.py +0 -0
  11. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/decorators.py +0 -0
  12. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/item.py +0 -0
  13. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/log.py +0 -0
  14. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/request.py +0 -0
  15. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/response.py +0 -0
  16. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/seed.py +0 -0
  17. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/constant.py +0 -0
  18. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/crawlers/__init__.py +0 -0
  19. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/crawlers/base_crawler.py +0 -0
  20. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/crawlers/crawler.py +0 -0
  21. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/crawlers/file_crawler.py +0 -0
  22. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/db/__init__.py +0 -0
  23. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/db/api_db.py +0 -0
  24. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/db/redis_db_new.py +0 -0
  25. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/exceptions/__init__.py +0 -0
  26. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/exceptions/oss_db_exception.py +0 -0
  27. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/__init__.py +0 -0
  28. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/launcher.py +0 -0
  29. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/launcher_air.py +0 -0
  30. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/launcher_api.py +0 -0
  31. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/pipelines/__init__.py +0 -0
  32. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/pipelines/pipeline.py +0 -0
  33. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/pipelines/pipeline_console.py +0 -0
  34. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/pipelines/pipeline_loghub.py +0 -0
  35. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/setting.py +0 -0
  36. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/__init__.py +0 -0
  37. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/bloom.py +0 -0
  38. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/dotting.py +0 -0
  39. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/oss.py +0 -0
  40. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/tools.py +0 -0
  41. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  42. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  43. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/requires.txt +0 -0
  44. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/top_level.txt +0 -0
  45. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/setup.cfg +0 -0
  46. {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/test/test.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.58
3
+ Version: 1.2.59
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
@@ -4,7 +4,7 @@ from cobweb import setting
4
4
  from redis.exceptions import ConnectionError, TimeoutError
5
5
 
6
6
 
7
- class RedisClient:
7
+ class RedisDB:
8
8
  def __init__(self, **kwargs):
9
9
  redis_config = kwargs or setting.REDIS_CONFIG
10
10
  self.host = redis_config['host']
@@ -147,4 +147,69 @@ class RedisClient:
147
147
 
148
148
  def execute_lua(self, lua_script: str, keys: list, *args):
149
149
  execute = self.execute_command("register_script", lua_script)
150
- return execute(keys=keys, args=args)
150
+ return execute(keys=keys, args=args)
151
+
152
+ def lock(self, key, t=15) -> bool:
153
+ lua_script = """
154
+ local status = redis.call('setnx', KEYS[1], 1)
155
+ if ( status == 1 ) then
156
+ redis.call('expire', KEYS[1], ARGV[1])
157
+ end
158
+ return status
159
+ """
160
+ status = self.execute_lua(lua_script, [key], t)
161
+ return bool(status)
162
+
163
+ def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
164
+ lua_script = """
165
+ local min = ARGV[1]
166
+ local max = ARGV[2]
167
+ local start = ARGV[3]
168
+ local count = ARGV[4]
169
+ local score = ARGV[5]
170
+ local members = nil
171
+
172
+ if ( type(count) == string ) then
173
+ members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
174
+ else
175
+ members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
176
+ end
177
+
178
+ local result = {}
179
+
180
+ for i = 1, #members, 2 do
181
+ local priority = nil
182
+ local member = members[i]
183
+ local originPriority = nil
184
+ if ( members[i+1] + 0 < 0 ) then
185
+ originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
186
+ else
187
+ originPriority = math.floor(members[i+1])
188
+ end
189
+
190
+ if ( score + 0 >= 1000 ) then
191
+ priority = -score - originPriority / 1000
192
+ elseif ( score + 0 == 0 ) then
193
+ priority = originPriority
194
+ else
195
+ originPriority = score
196
+ priority = score
197
+ end
198
+ redis.call('zadd', KEYS[1], priority, member)
199
+ table.insert(result, member)
200
+ table.insert(result, originPriority)
201
+ end
202
+
203
+ return result
204
+ """
205
+ members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
206
+ return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
207
+
208
+ def done(self, keys: list, *args) -> list:
209
+ lua_script = """
210
+ for i, member in ipairs(ARGV) do
211
+ redis.call("zrem", KEYS[1], member)
212
+ redis.call("sadd", KEYS[2], member)
213
+ end
214
+ """
215
+ self.execute_lua(lua_script, keys, *args)
@@ -1,7 +1,6 @@
1
1
  import time
2
2
  import threading
3
3
 
4
- from h5py.h5pl import insert
5
4
 
6
5
  from cobweb.db import RedisDB
7
6
  from cobweb.base import Seed, logger
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.58
3
+ Version: 1.2.59
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.2.58",
8
+ version="1.2.59",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",