appier 1.31.4__py2.py3-none-any.whl → 1.32.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- appier/__init__.py +333 -52
- appier/amqp.py +29 -30
- appier/api.py +214 -212
- appier/asgi.py +54 -55
- appier/async_neo.py +46 -35
- appier/async_old.py +55 -42
- appier/asynchronous.py +7 -13
- appier/base.py +1762 -1429
- appier/bus.py +51 -52
- appier/cache.py +99 -84
- appier/common.py +9 -11
- appier/component.py +17 -19
- appier/compress.py +25 -28
- appier/config.py +96 -73
- appier/controller.py +9 -15
- appier/crypt.py +25 -21
- appier/data.py +73 -57
- appier/defines.py +191 -226
- appier/exceptions.py +103 -63
- appier/execution.py +94 -88
- appier/export.py +90 -88
- appier/extra.py +6 -13
- appier/extra_neo.py +8 -11
- appier/extra_old.py +18 -16
- appier/geo.py +57 -47
- appier/git.py +101 -90
- appier/graph.py +23 -24
- appier/http.py +520 -398
- appier/legacy.py +373 -180
- appier/log.py +90 -97
- appier/meta.py +42 -42
- appier/mock.py +32 -34
- appier/model.py +793 -681
- appier/model_a.py +208 -183
- appier/mongo.py +183 -107
- appier/observer.py +39 -31
- appier/part.py +23 -24
- appier/preferences.py +44 -47
- appier/queuing.py +78 -96
- appier/redisdb.py +40 -35
- appier/request.py +227 -175
- appier/scheduler.py +13 -18
- appier/serialize.py +37 -31
- appier/session.py +161 -147
- appier/settings.py +2 -11
- appier/smtp.py +53 -49
- appier/storage.py +39 -33
- appier/structures.py +50 -45
- appier/test/__init__.py +2 -11
- appier/test/base.py +111 -108
- appier/test/cache.py +28 -35
- appier/test/config.py +10 -19
- appier/test/crypt.py +3 -12
- appier/test/data.py +3 -12
- appier/test/exceptions.py +8 -17
- appier/test/export.py +16 -33
- appier/test/graph.py +27 -60
- appier/test/http.py +42 -54
- appier/test/legacy.py +20 -30
- appier/test/log.py +14 -35
- appier/test/mock.py +27 -123
- appier/test/model.py +79 -91
- appier/test/part.py +5 -14
- appier/test/preferences.py +5 -13
- appier/test/queuing.py +29 -37
- appier/test/request.py +61 -73
- appier/test/serialize.py +12 -23
- appier/test/session.py +10 -19
- appier/test/smtp.py +8 -14
- appier/test/structures.py +20 -24
- appier/test/typesf.py +14 -28
- appier/test/util.py +480 -438
- appier/typesf.py +251 -171
- appier/util.py +578 -407
- appier/validation.py +280 -143
- {appier-1.31.4.dist-info → appier-1.32.0.dist-info}/METADATA +6 -1
- appier-1.32.0.dist-info/RECORD +86 -0
- appier-1.31.4.dist-info/RECORD +0 -86
- {appier-1.31.4.dist-info → appier-1.32.0.dist-info}/LICENSE +0 -0
- {appier-1.31.4.dist-info → appier-1.32.0.dist-info}/WHEEL +0 -0
- {appier-1.31.4.dist-info → appier-1.32.0.dist-info}/top_level.txt +0 -0
appier/part.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
# -*- coding: utf-8 -*-
|
|
3
3
|
|
|
4
4
|
# Hive Appier Framework
|
|
5
|
-
# Copyright (c) 2008-
|
|
5
|
+
# Copyright (c) 2008-2024 Hive Solutions Lda.
|
|
6
6
|
#
|
|
7
7
|
# This file is part of Hive Appier Framework.
|
|
8
8
|
#
|
|
@@ -22,16 +22,7 @@
|
|
|
22
22
|
__author__ = "João Magalhães <joamag@hive.pt>"
|
|
23
23
|
""" The author(s) of the module """
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
""" The version of the module """
|
|
27
|
-
|
|
28
|
-
__revision__ = "$LastChangedRevision$"
|
|
29
|
-
""" The revision number of the module """
|
|
30
|
-
|
|
31
|
-
__date__ = "$LastChangedDate$"
|
|
32
|
-
""" The last change date of the module """
|
|
33
|
-
|
|
34
|
-
__copyright__ = "Copyright (c) 2008-2022 Hive Solutions Lda."
|
|
25
|
+
__copyright__ = "Copyright (c) 2008-2024 Hive Solutions Lda."
|
|
35
26
|
""" The copyright for the module """
|
|
36
27
|
|
|
37
28
|
__license__ = "Apache License, Version 2.0"
|
|
@@ -43,6 +34,7 @@ import logging
|
|
|
43
34
|
|
|
44
35
|
from . import util
|
|
45
36
|
|
|
37
|
+
|
|
46
38
|
class Part(object):
|
|
47
39
|
"""
|
|
48
40
|
Abstract top level class for the "part" module infra-structure
|
|
@@ -53,11 +45,12 @@ class Part(object):
|
|
|
53
45
|
features to it in a "transparent" way.
|
|
54
46
|
"""
|
|
55
47
|
|
|
56
|
-
def __init__(self, owner
|
|
48
|
+
def __init__(self, owner=None, *args, **kwargs):
|
|
57
49
|
self.owner = owner
|
|
58
50
|
self.loaded = False
|
|
59
51
|
self._load_paths()
|
|
60
|
-
if owner:
|
|
52
|
+
if owner:
|
|
53
|
+
self.register(owner)
|
|
61
54
|
|
|
62
55
|
def __getattr__(self, name):
|
|
63
56
|
if self.owner and hasattr(self.owner, name):
|
|
@@ -66,17 +59,22 @@ class Part(object):
|
|
|
66
59
|
|
|
67
60
|
@classmethod
|
|
68
61
|
def _merge_paths(cls, first, second):
|
|
69
|
-
if not isinstance(first, (list, tuple)):
|
|
70
|
-
|
|
71
|
-
if isinstance(
|
|
72
|
-
|
|
62
|
+
if not isinstance(first, (list, tuple)):
|
|
63
|
+
first = [first]
|
|
64
|
+
if not isinstance(second, (list, tuple)):
|
|
65
|
+
second = [second]
|
|
66
|
+
if isinstance(first, tuple):
|
|
67
|
+
first = list(first)
|
|
68
|
+
if isinstance(second, tuple):
|
|
69
|
+
second = list(second)
|
|
73
70
|
return first + second
|
|
74
71
|
|
|
75
72
|
def name(self):
|
|
76
73
|
cls = self.__class__
|
|
77
74
|
cls_name = cls.__name__
|
|
78
75
|
name = util.camel_to_underscore(cls_name)
|
|
79
|
-
if name.endswith("_part"):
|
|
76
|
+
if name.endswith("_part"):
|
|
77
|
+
name = name[:-5]
|
|
80
78
|
return name
|
|
81
79
|
|
|
82
80
|
def version(self):
|
|
@@ -84,14 +82,13 @@ class Part(object):
|
|
|
84
82
|
|
|
85
83
|
def info(self):
|
|
86
84
|
return dict(
|
|
87
|
-
name = self.
|
|
88
|
-
version = self.version(),
|
|
89
|
-
class_name = self.class_name()
|
|
85
|
+
name=self.name(), version=self.version(), class_name=self.class_name()
|
|
90
86
|
)
|
|
91
87
|
|
|
92
88
|
def class_name(self):
|
|
93
89
|
cls = self.__class__
|
|
94
|
-
if not self.__module__:
|
|
90
|
+
if not self.__module__:
|
|
91
|
+
return cls.__name__
|
|
95
92
|
return self.__module__ + "." + cls.__name__
|
|
96
93
|
|
|
97
94
|
def register(self, owner):
|
|
@@ -119,8 +116,10 @@ class Part(object):
|
|
|
119
116
|
|
|
120
117
|
@property
|
|
121
118
|
def logger(self):
|
|
122
|
-
if self.owner:
|
|
123
|
-
|
|
119
|
+
if self.owner:
|
|
120
|
+
return self.owner.logger
|
|
121
|
+
else:
|
|
122
|
+
return logging.getLogger()
|
|
124
123
|
|
|
125
124
|
def _load_paths(self):
|
|
126
125
|
module = self.__class__.__module__
|
appier/preferences.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
# -*- coding: utf-8 -*-
|
|
3
3
|
|
|
4
4
|
# Hive Appier Framework
|
|
5
|
-
# Copyright (c) 2008-
|
|
5
|
+
# Copyright (c) 2008-2024 Hive Solutions Lda.
|
|
6
6
|
#
|
|
7
7
|
# This file is part of Hive Appier Framework.
|
|
8
8
|
#
|
|
@@ -22,16 +22,7 @@
|
|
|
22
22
|
__author__ = "João Magalhães <joamag@hive.pt>"
|
|
23
23
|
""" The author(s) of the module """
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
""" The version of the module """
|
|
27
|
-
|
|
28
|
-
__revision__ = "$LastChangedRevision$"
|
|
29
|
-
""" The revision number of the module """
|
|
30
|
-
|
|
31
|
-
__date__ = "$LastChangedDate$"
|
|
32
|
-
""" The last change date of the module """
|
|
33
|
-
|
|
34
|
-
__copyright__ = "Copyright (c) 2008-2022 Hive Solutions Lda."
|
|
25
|
+
__copyright__ = "Copyright (c) 2008-2024 Hive Solutions Lda."
|
|
35
26
|
""" The copyright for the module """
|
|
36
27
|
|
|
37
28
|
__license__ = "Apache License, Version 2.0"
|
|
@@ -48,15 +39,16 @@ from . import redisdb
|
|
|
48
39
|
from . import component
|
|
49
40
|
from . import exceptions
|
|
50
41
|
|
|
51
|
-
class Preferences(component.Component):
|
|
52
42
|
|
|
53
|
-
|
|
54
|
-
|
|
43
|
+
class Preferences(component.Component):
|
|
44
|
+
def __init__(self, name="preferences", owner=None, *args, **kwargs):
|
|
45
|
+
component.Component.__init__(self, name=name, owner=owner, *args, **kwargs)
|
|
55
46
|
load = kwargs.pop("load", True)
|
|
56
|
-
if load:
|
|
47
|
+
if load:
|
|
48
|
+
self.load(*args, **kwargs)
|
|
57
49
|
|
|
58
50
|
def __getitem__(self, key):
|
|
59
|
-
return self.get(key, strict
|
|
51
|
+
return self.get(key, strict=True)
|
|
60
52
|
|
|
61
53
|
def __setitem__(self, key, value):
|
|
62
54
|
self.set(key, value)
|
|
@@ -68,13 +60,14 @@ class Preferences(component.Component):
|
|
|
68
60
|
def new(cls, *args, **kwargs):
|
|
69
61
|
return cls(*args, **kwargs)
|
|
70
62
|
|
|
71
|
-
def get(self, name, default
|
|
72
|
-
return self._get(name, default
|
|
63
|
+
def get(self, name, default=None, strict=False, *args, **kwargs):
|
|
64
|
+
return self._get(name, default=default, strict=strict, *args, **kwargs)
|
|
73
65
|
|
|
74
66
|
def set(self, name, value, *args, **kwargs):
|
|
75
67
|
flush = kwargs.get("flush", True)
|
|
76
68
|
result = self._set(name, value, *args, **kwargs)
|
|
77
|
-
if flush:
|
|
69
|
+
if flush:
|
|
70
|
+
self.flush()
|
|
78
71
|
return result
|
|
79
72
|
|
|
80
73
|
def delete(self, name, *args, **kwargs):
|
|
@@ -86,7 +79,7 @@ class Preferences(component.Component):
|
|
|
86
79
|
def clear(self, *args, **kwargs):
|
|
87
80
|
return self._clear(*args, **kwargs)
|
|
88
81
|
|
|
89
|
-
def _get(self, name, default
|
|
82
|
+
def _get(self, name, default=None, strict=False, *args, **kwargs):
|
|
90
83
|
raise exceptions.NotImplementedError()
|
|
91
84
|
|
|
92
85
|
def _set(self, name, value, *args, **kwargs):
|
|
@@ -101,10 +94,10 @@ class Preferences(component.Component):
|
|
|
101
94
|
def _clear(self, *args, **kwargs):
|
|
102
95
|
raise exceptions.NotImplementedError()
|
|
103
96
|
|
|
104
|
-
class MemoryPreferences(Preferences):
|
|
105
97
|
|
|
106
|
-
|
|
107
|
-
|
|
98
|
+
class MemoryPreferences(Preferences):
|
|
99
|
+
def __init__(self, name="memory", owner=None, *args, **kwargs):
|
|
100
|
+
Preferences.__init__(self, name=name, owner=owner, *args, **kwargs)
|
|
108
101
|
|
|
109
102
|
def _load(self, *args, **kwargs):
|
|
110
103
|
Preferences._load(self, *args, **kwargs)
|
|
@@ -114,8 +107,9 @@ class MemoryPreferences(Preferences):
|
|
|
114
107
|
Preferences._unload(self, *args, **kwargs)
|
|
115
108
|
self._preferences = None
|
|
116
109
|
|
|
117
|
-
def _get(self, name, default
|
|
118
|
-
if strict:
|
|
110
|
+
def _get(self, name, default=None, strict=False, *args, **kwargs):
|
|
111
|
+
if strict:
|
|
112
|
+
return self._preferences[name]
|
|
119
113
|
return self._preferences.get(name, default)
|
|
120
114
|
|
|
121
115
|
def _set(self, name, value, *args, **kwargs):
|
|
@@ -130,10 +124,10 @@ class MemoryPreferences(Preferences):
|
|
|
130
124
|
def _clear(self, *args, **kwargs):
|
|
131
125
|
self._preferences.clear()
|
|
132
126
|
|
|
133
|
-
class FilePreferences(Preferences):
|
|
134
127
|
|
|
135
|
-
|
|
136
|
-
|
|
128
|
+
class FilePreferences(Preferences):
|
|
129
|
+
def __init__(self, name="file", owner=None, *args, **kwargs):
|
|
130
|
+
Preferences.__init__(self, name=name, owner=owner, *args, **kwargs)
|
|
137
131
|
|
|
138
132
|
def _load(self, *args, **kwargs):
|
|
139
133
|
Preferences._load(self, *args, **kwargs)
|
|
@@ -144,8 +138,9 @@ class FilePreferences(Preferences):
|
|
|
144
138
|
Preferences._unload(self, *args, **kwargs)
|
|
145
139
|
self._close()
|
|
146
140
|
|
|
147
|
-
def _get(self, name, default
|
|
148
|
-
if strict:
|
|
141
|
+
def _get(self, name, default=None, strict=False, *args, **kwargs):
|
|
142
|
+
if strict:
|
|
143
|
+
return self._shelve[name]
|
|
149
144
|
return self._shelve.get(name, default)
|
|
150
145
|
|
|
151
146
|
def _set(self, name, value, *args, **kwargs):
|
|
@@ -158,26 +153,25 @@ class FilePreferences(Preferences):
|
|
|
158
153
|
self._sync()
|
|
159
154
|
|
|
160
155
|
def _clear(self, *args, **kwargs):
|
|
161
|
-
if not os.path.exists(self.preferences_path):
|
|
156
|
+
if not os.path.exists(self.preferences_path):
|
|
157
|
+
return
|
|
162
158
|
os.remove(self.preferences_path)
|
|
163
159
|
|
|
164
160
|
def _open(self):
|
|
165
161
|
self._ensure_path()
|
|
166
|
-
self._shelve = shelve.open(
|
|
167
|
-
self.preferences_path,
|
|
168
|
-
protocol = 2,
|
|
169
|
-
writeback = True
|
|
170
|
-
)
|
|
162
|
+
self._shelve = shelve.open(self.preferences_path, protocol=2, writeback=True)
|
|
171
163
|
|
|
172
164
|
def _close(self):
|
|
173
|
-
if not self._shelve:
|
|
165
|
+
if not self._shelve:
|
|
166
|
+
return
|
|
174
167
|
self._shelve.close()
|
|
175
168
|
self._shelve = None
|
|
176
169
|
|
|
177
170
|
def _ensure_path(self):
|
|
178
|
-
if self.preferences_path:
|
|
171
|
+
if self.preferences_path:
|
|
172
|
+
return
|
|
179
173
|
app_path = common.base().get_base_path()
|
|
180
|
-
util.verify(not app_path == None, message
|
|
174
|
+
util.verify(not app_path == None, message="No app path available")
|
|
181
175
|
preferences_path = os.path.join(app_path, "preferences.shelve")
|
|
182
176
|
preferences_path = config.conf("PREFERENCES_PATH", preferences_path)
|
|
183
177
|
preferences_path = os.path.expanduser(preferences_path)
|
|
@@ -185,7 +179,7 @@ class FilePreferences(Preferences):
|
|
|
185
179
|
preferences_path = os.path.normpath(preferences_path)
|
|
186
180
|
self.preferences_path = preferences_path
|
|
187
181
|
|
|
188
|
-
def _sync(self, secure
|
|
182
|
+
def _sync(self, secure=None):
|
|
189
183
|
if secure == None:
|
|
190
184
|
secure = self._db_secure()
|
|
191
185
|
if secure:
|
|
@@ -202,8 +196,8 @@ class FilePreferences(Preferences):
|
|
|
202
196
|
shelve_dbm = shelve_cls.__name__
|
|
203
197
|
return shelve_dbm
|
|
204
198
|
|
|
205
|
-
class RedisPreferences(Preferences):
|
|
206
199
|
|
|
200
|
+
class RedisPreferences(Preferences):
|
|
207
201
|
SERIALIZER = pickle
|
|
208
202
|
""" The serializer to be used for the values contained in
|
|
209
203
|
the preferences (used on top of the class) """
|
|
@@ -212,8 +206,8 @@ class RedisPreferences(Preferences):
|
|
|
212
206
|
"""The default prefix value that is going to
|
|
213
207
|
be prepended to the key name to handle preferences"""
|
|
214
208
|
|
|
215
|
-
def __init__(self, name
|
|
216
|
-
Preferences.__init__(self, name
|
|
209
|
+
def __init__(self, name="redis", owner=None, *args, **kwargs):
|
|
210
|
+
Preferences.__init__(self, name=name, owner=owner, *args, **kwargs)
|
|
217
211
|
|
|
218
212
|
def _load(self, *args, **kwargs):
|
|
219
213
|
Preferences._load(self, *args, **kwargs)
|
|
@@ -226,14 +220,17 @@ class RedisPreferences(Preferences):
|
|
|
226
220
|
Preferences._unload(self, *args, **kwargs)
|
|
227
221
|
self._close()
|
|
228
222
|
|
|
229
|
-
def _get(self, name, default
|
|
223
|
+
def _get(self, name, default=None, strict=False, *args, **kwargs):
|
|
230
224
|
name = self._fqn(name)
|
|
231
225
|
value = self._redis.get(name)
|
|
232
226
|
if not value:
|
|
233
|
-
if strict:
|
|
227
|
+
if strict:
|
|
228
|
+
raise KeyError("not found")
|
|
229
|
+
return default
|
|
230
|
+
try:
|
|
231
|
+
return self._serializer.loads(value)
|
|
232
|
+
except Exception:
|
|
234
233
|
return default
|
|
235
|
-
try: return self._serializer.loads(value)
|
|
236
|
-
except Exception: return default
|
|
237
234
|
|
|
238
235
|
def _set(self, name, value, *args, **kwargs):
|
|
239
236
|
name = self._fqn(name)
|
appier/queuing.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
# -*- coding: utf-8 -*-
|
|
3
3
|
|
|
4
4
|
# Hive Appier Framework
|
|
5
|
-
# Copyright (c) 2008-
|
|
5
|
+
# Copyright (c) 2008-2024 Hive Solutions Lda.
|
|
6
6
|
#
|
|
7
7
|
# This file is part of Hive Appier Framework.
|
|
8
8
|
#
|
|
@@ -22,16 +22,7 @@
|
|
|
22
22
|
__author__ = "João Magalhães <joamag@hive.pt>"
|
|
23
23
|
""" The author(s) of the module """
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
""" The version of the module """
|
|
27
|
-
|
|
28
|
-
__revision__ = "$LastChangedRevision$"
|
|
29
|
-
""" The revision number of the module """
|
|
30
|
-
|
|
31
|
-
__date__ = "$LastChangedDate$"
|
|
32
|
-
""" The last change date of the module """
|
|
33
|
-
|
|
34
|
-
__copyright__ = "Copyright (c) 2008-2022 Hive Solutions Lda."
|
|
25
|
+
__copyright__ = "Copyright (c) 2008-2024 Hive Solutions Lda."
|
|
35
26
|
""" The copyright for the module """
|
|
36
27
|
|
|
37
28
|
__license__ = "Apache License, Version 2.0"
|
|
@@ -46,21 +37,21 @@ from . import amqp
|
|
|
46
37
|
from . import legacy
|
|
47
38
|
from . import exceptions
|
|
48
39
|
|
|
49
|
-
class Queue(object):
|
|
50
40
|
|
|
41
|
+
class Queue(object):
|
|
51
42
|
def length(self):
|
|
52
43
|
raise exceptions.NotImplementedError()
|
|
53
44
|
|
|
54
45
|
def clear(self):
|
|
55
46
|
raise exceptions.NotImplementedError()
|
|
56
47
|
|
|
57
|
-
def push(self, value, priority
|
|
48
|
+
def push(self, value, priority=None, identify=False):
|
|
58
49
|
raise exceptions.NotImplementedError()
|
|
59
50
|
|
|
60
|
-
def pop(self, block
|
|
51
|
+
def pop(self, block=True, full=False):
|
|
61
52
|
raise exceptions.NotImplementedError()
|
|
62
53
|
|
|
63
|
-
def subscribe(self, callback, full
|
|
54
|
+
def subscribe(self, callback, full=False):
|
|
64
55
|
raise exceptions.NotImplementedError()
|
|
65
56
|
|
|
66
57
|
def loop(self):
|
|
@@ -75,23 +66,20 @@ class Queue(object):
|
|
|
75
66
|
def nack(self):
|
|
76
67
|
raise exceptions.NotImplementedError()
|
|
77
68
|
|
|
78
|
-
def build_value(
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
if identify: identifier = self.build_identifier()
|
|
86
|
-
else: identifier = None
|
|
87
|
-
if priority and reverse: priority *= -1
|
|
69
|
+
def build_value(self, value, priority=None, identify=False, reverse=False):
|
|
70
|
+
if identify:
|
|
71
|
+
identifier = self.build_identifier()
|
|
72
|
+
else:
|
|
73
|
+
identifier = None
|
|
74
|
+
if priority and reverse:
|
|
75
|
+
priority *= -1
|
|
88
76
|
return (priority, identifier, value), identifier
|
|
89
77
|
|
|
90
78
|
def build_identifier(self):
|
|
91
79
|
return str(uuid.uuid4())
|
|
92
80
|
|
|
93
|
-
class MemoryQueue(Queue):
|
|
94
81
|
|
|
82
|
+
class MemoryQueue(Queue):
|
|
95
83
|
def __init__(self):
|
|
96
84
|
Queue.__init__(self)
|
|
97
85
|
self._queue = []
|
|
@@ -102,25 +90,24 @@ class MemoryQueue(Queue):
|
|
|
102
90
|
def clear(self):
|
|
103
91
|
del self._queue[:]
|
|
104
92
|
|
|
105
|
-
def push(self, value, priority
|
|
93
|
+
def push(self, value, priority=None, identify=False):
|
|
106
94
|
value, identifier = self.build_value(
|
|
107
|
-
value,
|
|
108
|
-
priority = priority,
|
|
109
|
-
identify = identify,
|
|
110
|
-
reverse = True
|
|
95
|
+
value, priority=priority, identify=identify, reverse=True
|
|
111
96
|
)
|
|
112
97
|
heapq.heappush(self._queue, value)
|
|
113
98
|
return identifier
|
|
114
99
|
|
|
115
|
-
def pop(self, block
|
|
100
|
+
def pop(self, block=True, full=False):
|
|
116
101
|
priority, identifier, value = heapq.heappop(self._queue)
|
|
117
102
|
return (priority, identifier, value) if full else value
|
|
118
103
|
|
|
119
|
-
class MultiprocessQueue(Queue):
|
|
120
104
|
|
|
105
|
+
class MultiprocessQueue(Queue):
|
|
121
106
|
def __init__(self):
|
|
122
|
-
try:
|
|
123
|
-
|
|
107
|
+
try:
|
|
108
|
+
import queue
|
|
109
|
+
except ImportError:
|
|
110
|
+
import Queue as queue
|
|
124
111
|
Queue.__init__(self)
|
|
125
112
|
self._queue = queue.PriorityQueue()
|
|
126
113
|
|
|
@@ -128,36 +115,35 @@ class MultiprocessQueue(Queue):
|
|
|
128
115
|
return self._queue.qsize()
|
|
129
116
|
|
|
130
117
|
def clear(self):
|
|
131
|
-
try:
|
|
132
|
-
|
|
118
|
+
try:
|
|
119
|
+
import queue
|
|
120
|
+
except ImportError:
|
|
121
|
+
import Queue as queue
|
|
133
122
|
self._queue = queue.PriorityQueue()
|
|
134
123
|
|
|
135
|
-
def push(self, value, priority
|
|
124
|
+
def push(self, value, priority=None, identify=False):
|
|
136
125
|
value, identifier = self.build_value(
|
|
137
|
-
value,
|
|
138
|
-
priority = priority,
|
|
139
|
-
identify = identify,
|
|
140
|
-
reverse = True
|
|
126
|
+
value, priority=priority, identify=identify, reverse=True
|
|
141
127
|
)
|
|
142
128
|
self._queue.put(value)
|
|
143
129
|
return identifier
|
|
144
130
|
|
|
145
|
-
def pop(self, block
|
|
131
|
+
def pop(self, block=True, full=False):
|
|
146
132
|
priority, identifier, value = self._queue.get(block)
|
|
147
133
|
return (priority, identifier, value) if full else value
|
|
148
134
|
|
|
149
|
-
class AMQPQueue(Queue):
|
|
150
135
|
|
|
136
|
+
class AMQPQueue(Queue):
|
|
151
137
|
def __init__(
|
|
152
138
|
self,
|
|
153
|
-
url
|
|
154
|
-
name
|
|
155
|
-
durable
|
|
156
|
-
max_priority
|
|
157
|
-
encoder
|
|
158
|
-
protocol
|
|
159
|
-
encoding
|
|
160
|
-
amqp
|
|
139
|
+
url=None,
|
|
140
|
+
name="default",
|
|
141
|
+
durable=False,
|
|
142
|
+
max_priority=255,
|
|
143
|
+
encoder="pickle",
|
|
144
|
+
protocol=2,
|
|
145
|
+
encoding="utf-8",
|
|
146
|
+
amqp=None,
|
|
161
147
|
):
|
|
162
148
|
self.url = url
|
|
163
149
|
self.name = name
|
|
@@ -170,33 +156,27 @@ class AMQPQueue(Queue):
|
|
|
170
156
|
self._build()
|
|
171
157
|
|
|
172
158
|
def clear(self):
|
|
173
|
-
self.channel.queue_purge(queue
|
|
159
|
+
self.channel.queue_purge(queue=self.name)
|
|
174
160
|
|
|
175
|
-
def push(self, value, priority
|
|
161
|
+
def push(self, value, priority=None, identify=False):
|
|
176
162
|
value, identifier = self.build_value(
|
|
177
|
-
value,
|
|
178
|
-
priority = priority,
|
|
179
|
-
identify = identify,
|
|
180
|
-
reverse = False
|
|
163
|
+
value, priority=priority, identify=identify, reverse=False
|
|
181
164
|
)
|
|
182
165
|
body = self._dump(value)
|
|
183
166
|
|
|
184
167
|
self._add_callback(
|
|
185
168
|
self.channel.basic_publish,
|
|
186
|
-
exchange
|
|
187
|
-
routing_key
|
|
188
|
-
body
|
|
189
|
-
properties
|
|
190
|
-
delivery_mode = 2,
|
|
191
|
-
priority = value[0] or 0
|
|
192
|
-
)
|
|
169
|
+
exchange="",
|
|
170
|
+
routing_key=self.name,
|
|
171
|
+
body=body,
|
|
172
|
+
properties=amqp.properties(delivery_mode=2, priority=value[0] or 0),
|
|
193
173
|
)
|
|
194
174
|
|
|
195
175
|
return identifier
|
|
196
176
|
|
|
197
|
-
def pop(self, block
|
|
177
|
+
def pop(self, block=True, full=False):
|
|
198
178
|
_method, _properties, body = self.channel.basic_get(
|
|
199
|
-
queue
|
|
179
|
+
queue=self.name, auto_ack=False
|
|
200
180
|
)
|
|
201
181
|
priority, identifier, value = self._load(body)
|
|
202
182
|
return (priority, identifier, value) if full else value
|
|
@@ -204,33 +184,33 @@ class AMQPQueue(Queue):
|
|
|
204
184
|
def subscribe(
|
|
205
185
|
self,
|
|
206
186
|
callback,
|
|
207
|
-
full
|
|
208
|
-
auto_ack
|
|
209
|
-
exclusive
|
|
210
|
-
consumer_tag
|
|
211
|
-
arguments
|
|
187
|
+
full=False,
|
|
188
|
+
auto_ack=True,
|
|
189
|
+
exclusive=False,
|
|
190
|
+
consumer_tag=None,
|
|
191
|
+
arguments=None,
|
|
212
192
|
):
|
|
213
193
|
def handler(channel, method, properties, body):
|
|
214
194
|
priority, identifier, value = self._load(body)
|
|
215
195
|
result = (priority, identifier, value) if full else value
|
|
216
|
-
ack = lambda: self.ack(delivery_tag
|
|
217
|
-
nack = lambda: self.nack(delivery_tag
|
|
196
|
+
ack = lambda: self.ack(delivery_tag=method.delivery_tag)
|
|
197
|
+
nack = lambda: self.nack(delivery_tag=method.delivery_tag)
|
|
218
198
|
callback(result) if auto_ack else callback(result, ack, nack)
|
|
219
199
|
|
|
220
200
|
self.channel.basic_consume(
|
|
221
|
-
queue
|
|
222
|
-
on_message_callback
|
|
223
|
-
auto_ack
|
|
224
|
-
exclusive
|
|
225
|
-
consumer_tag
|
|
226
|
-
arguments
|
|
201
|
+
queue=self.name,
|
|
202
|
+
on_message_callback=handler,
|
|
203
|
+
auto_ack=auto_ack,
|
|
204
|
+
exclusive=exclusive,
|
|
205
|
+
consumer_tag=consumer_tag,
|
|
206
|
+
arguments=arguments,
|
|
227
207
|
)
|
|
228
208
|
|
|
229
|
-
def ack(self, delivery_tag
|
|
230
|
-
self.channel.basic_ack(delivery_tag
|
|
209
|
+
def ack(self, delivery_tag=None):
|
|
210
|
+
self.channel.basic_ack(delivery_tag=delivery_tag)
|
|
231
211
|
|
|
232
|
-
def nack(self, delivery_tag
|
|
233
|
-
self.channel.basic_nack(delivery_tag
|
|
212
|
+
def nack(self, delivery_tag=None):
|
|
213
|
+
self.channel.basic_nack(delivery_tag=delivery_tag)
|
|
234
214
|
|
|
235
215
|
def loop(self):
|
|
236
216
|
self.channel.start_consuming()
|
|
@@ -242,35 +222,37 @@ class AMQPQueue(Queue):
|
|
|
242
222
|
return self._dumper(value)
|
|
243
223
|
|
|
244
224
|
def _dump_pickle(self, value):
|
|
245
|
-
return legacy.cPickle.dumps(value, protocol
|
|
225
|
+
return legacy.cPickle.dumps(value, protocol=self.protocol)
|
|
246
226
|
|
|
247
227
|
def _dump_json(self, value):
|
|
248
228
|
body = json.dumps(value)
|
|
249
|
-
return legacy.bytes(body, encoding
|
|
229
|
+
return legacy.bytes(body, encoding=self.encoding)
|
|
250
230
|
|
|
251
231
|
def _load(self, body):
|
|
252
232
|
return self._loader(body)
|
|
253
233
|
|
|
254
234
|
def _load_pickle(self, body):
|
|
255
|
-
if legacy.PYTHON_3:
|
|
256
|
-
|
|
235
|
+
if legacy.PYTHON_3:
|
|
236
|
+
kwargs = dict(encoding="bytes")
|
|
237
|
+
else:
|
|
238
|
+
kwargs = dict()
|
|
257
239
|
return legacy.cPickle.loads(body, **kwargs)
|
|
258
240
|
|
|
259
241
|
def _load_json(self, body):
|
|
260
|
-
if legacy.is_bytes(body):
|
|
242
|
+
if legacy.is_bytes(body):
|
|
243
|
+
body = body.decode(self.encoding)
|
|
261
244
|
return json.loads(body)
|
|
262
245
|
|
|
263
246
|
def _build(self):
|
|
264
|
-
if not self.amqp:
|
|
247
|
+
if not self.amqp:
|
|
248
|
+
self.amqp = amqp.AMQP(url=self.url)
|
|
265
249
|
self.connection = self.amqp.get_connection()
|
|
266
250
|
self.channel = self.connection.channel()
|
|
267
|
-
self.channel.basic_qos(prefetch_count
|
|
251
|
+
self.channel.basic_qos(prefetch_count=1)
|
|
268
252
|
self.queue = self.channel.queue_declare(
|
|
269
|
-
queue
|
|
270
|
-
durable
|
|
271
|
-
arguments
|
|
272
|
-
"x-max-priority" : self.max_priority
|
|
273
|
-
}
|
|
253
|
+
queue=self.name,
|
|
254
|
+
durable=self.durable,
|
|
255
|
+
arguments={"x-max-priority": self.max_priority},
|
|
274
256
|
)
|
|
275
257
|
self._dumper = getattr(self, "_dump_" + self.encoder)
|
|
276
258
|
self._loader = getattr(self, "_load_" + self.encoder)
|