cobweb-launcher 1.2.58__tar.gz → 1.2.59__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cobweb-launcher might be problematic. Click here for more details.
- {cobweb-launcher-1.2.58/cobweb_launcher.egg-info → cobweb-launcher-1.2.59}/PKG-INFO +1 -1
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/db/redis_db.py +67 -2
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/launcher_pro.py +0 -1
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59/cobweb_launcher.egg-info}/PKG-INFO +1 -1
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/setup.py +1 -1
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/LICENSE +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/README.md +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/__init__.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/__init__.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/common_queue.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/decorators.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/item.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/log.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/request.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/response.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/base/seed.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/constant.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/crawlers/__init__.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/crawlers/base_crawler.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/crawlers/crawler.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/crawlers/file_crawler.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/db/__init__.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/db/api_db.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/db/redis_db_new.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/exceptions/__init__.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/exceptions/oss_db_exception.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/__init__.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/launcher.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/launcher_air.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/launchers/launcher_api.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/pipelines/__init__.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/pipelines/pipeline.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/pipelines/pipeline_console.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/pipelines/pipeline_loghub.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/setting.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/__init__.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/bloom.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/dotting.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/oss.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb/utils/tools.py +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/requires.txt +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/top_level.txt +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/setup.cfg +0 -0
- {cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/test/test.py +0 -0
|
@@ -4,7 +4,7 @@ from cobweb import setting
|
|
|
4
4
|
from redis.exceptions import ConnectionError, TimeoutError
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
class
|
|
7
|
+
class RedisDB:
|
|
8
8
|
def __init__(self, **kwargs):
|
|
9
9
|
redis_config = kwargs or setting.REDIS_CONFIG
|
|
10
10
|
self.host = redis_config['host']
|
|
@@ -147,4 +147,69 @@ class RedisClient:
|
|
|
147
147
|
|
|
148
148
|
def execute_lua(self, lua_script: str, keys: list, *args):
|
|
149
149
|
execute = self.execute_command("register_script", lua_script)
|
|
150
|
-
return execute(keys=keys, args=args)
|
|
150
|
+
return execute(keys=keys, args=args)
|
|
151
|
+
|
|
152
|
+
def lock(self, key, t=15) -> bool:
|
|
153
|
+
lua_script = """
|
|
154
|
+
local status = redis.call('setnx', KEYS[1], 1)
|
|
155
|
+
if ( status == 1 ) then
|
|
156
|
+
redis.call('expire', KEYS[1], ARGV[1])
|
|
157
|
+
end
|
|
158
|
+
return status
|
|
159
|
+
"""
|
|
160
|
+
status = self.execute_lua(lua_script, [key], t)
|
|
161
|
+
return bool(status)
|
|
162
|
+
|
|
163
|
+
def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
|
|
164
|
+
lua_script = """
|
|
165
|
+
local min = ARGV[1]
|
|
166
|
+
local max = ARGV[2]
|
|
167
|
+
local start = ARGV[3]
|
|
168
|
+
local count = ARGV[4]
|
|
169
|
+
local score = ARGV[5]
|
|
170
|
+
local members = nil
|
|
171
|
+
|
|
172
|
+
if ( type(count) == string ) then
|
|
173
|
+
members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
|
|
174
|
+
else
|
|
175
|
+
members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
local result = {}
|
|
179
|
+
|
|
180
|
+
for i = 1, #members, 2 do
|
|
181
|
+
local priority = nil
|
|
182
|
+
local member = members[i]
|
|
183
|
+
local originPriority = nil
|
|
184
|
+
if ( members[i+1] + 0 < 0 ) then
|
|
185
|
+
originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
|
|
186
|
+
else
|
|
187
|
+
originPriority = math.floor(members[i+1])
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
if ( score + 0 >= 1000 ) then
|
|
191
|
+
priority = -score - originPriority / 1000
|
|
192
|
+
elseif ( score + 0 == 0 ) then
|
|
193
|
+
priority = originPriority
|
|
194
|
+
else
|
|
195
|
+
originPriority = score
|
|
196
|
+
priority = score
|
|
197
|
+
end
|
|
198
|
+
redis.call('zadd', KEYS[1], priority, member)
|
|
199
|
+
table.insert(result, member)
|
|
200
|
+
table.insert(result, originPriority)
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
return result
|
|
204
|
+
"""
|
|
205
|
+
members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
|
|
206
|
+
return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
|
|
207
|
+
|
|
208
|
+
def done(self, keys: list, *args) -> list:
|
|
209
|
+
lua_script = """
|
|
210
|
+
for i, member in ipairs(ARGV) do
|
|
211
|
+
redis.call("zrem", KEYS[1], member)
|
|
212
|
+
redis.call("sadd", KEYS[2], member)
|
|
213
|
+
end
|
|
214
|
+
"""
|
|
215
|
+
self.execute_lua(lua_script, keys, *args)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cobweb-launcher-1.2.58 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|