dsmq 0.7.0__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsmq/client.py +65 -0
- dsmq/demo.py +22 -0
- dsmq/example_get_client.py +2 -2
- dsmq/example_put_client.py +2 -2
- dsmq/{dsmq.py → server.py} +69 -103
- dsmq/tests/__init__.py +0 -0
- dsmq/tests/integration_test.py +238 -0
- {dsmq-0.7.0.dist-info → dsmq-1.1.0.dist-info}/METADATA +44 -12
- dsmq-1.1.0.dist-info/RECORD +12 -0
- {dsmq-0.7.0.dist-info → dsmq-1.1.0.dist-info}/WHEEL +1 -1
- dsmq/demo_linux.py +0 -25
- dsmq/example_server.py +0 -3
- dsmq-0.7.0.dist-info/RECORD +0 -10
- {dsmq-0.7.0.dist-info → dsmq-1.1.0.dist-info}/licenses/LICENSE +0 -0
dsmq/client.py
ADDED
@@ -0,0 +1,65 @@
|
|
1
|
+
import json
|
2
|
+
import time
|
3
|
+
from websockets.sync.client import connect as ws_connect
|
4
|
+
|
5
|
+
_default_host = "127.0.0.1"
|
6
|
+
_default_port = 30008
|
7
|
+
|
8
|
+
_n_retries = 10
|
9
|
+
_initial_retry = 0.01 # seconds
|
10
|
+
|
11
|
+
|
12
|
+
def connect(host=_default_host, port=_default_port):
|
13
|
+
return DSMQClientSideConnection(host, port)
|
14
|
+
|
15
|
+
|
16
|
+
class DSMQClientSideConnection:
|
17
|
+
def __init__(self, host, port):
|
18
|
+
self.uri = f"ws://{host}:{port}"
|
19
|
+
print(f"Connecting to dsmq server at {self.uri}")
|
20
|
+
for i_retry in range(_n_retries):
|
21
|
+
try:
|
22
|
+
self.websocket = ws_connect(self.uri)
|
23
|
+
break
|
24
|
+
except ConnectionRefusedError:
|
25
|
+
self.websocket = None
|
26
|
+
# Exponential backoff
|
27
|
+
# Wait twice as long each time before trying again.
|
28
|
+
time.sleep(_initial_retry * 2**i_retry)
|
29
|
+
print(" ...trying again")
|
30
|
+
|
31
|
+
if self.websocket is None:
|
32
|
+
raise ConnectionRefusedError("Could not connect to dsmq server.")
|
33
|
+
|
34
|
+
self.time_of_last_request = time.time()
|
35
|
+
|
36
|
+
def get(self, topic):
|
37
|
+
msg = {"action": "get", "topic": topic}
|
38
|
+
self.websocket.send(json.dumps(msg))
|
39
|
+
msg_text = self.websocket.recv()
|
40
|
+
msg = json.loads(msg_text)
|
41
|
+
return msg["message"]
|
42
|
+
|
43
|
+
def get_wait(self, topic):
|
44
|
+
"""
|
45
|
+
A variant of `get()` that retries a few times until it gets
|
46
|
+
a non-empty message. Adjust `_n_tries` and `_initial_retry`
|
47
|
+
to change how persistent it will be.
|
48
|
+
"""
|
49
|
+
for i_retry in range(_n_retries):
|
50
|
+
message = self.get(topic)
|
51
|
+
if message != "":
|
52
|
+
return message
|
53
|
+
time.sleep(_initial_retry * 2**i_retry)
|
54
|
+
return message
|
55
|
+
|
56
|
+
def put(self, topic, msg_body):
|
57
|
+
msg_dict = {"action": "put", "topic": topic, "message": msg_body}
|
58
|
+
self.websocket.send(json.dumps(msg_dict))
|
59
|
+
|
60
|
+
def shutdown_server(self):
|
61
|
+
msg_dict = {"action": "shutdown", "topic": ""}
|
62
|
+
self.websocket.send(json.dumps(msg_dict))
|
63
|
+
|
64
|
+
def close(self):
|
65
|
+
self.websocket.close()
|
dsmq/demo.py
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
import multiprocessing as mp
|
2
|
+
from dsmq.server import serve
|
3
|
+
import dsmq.example_get_client
|
4
|
+
import dsmq.example_put_client
|
5
|
+
|
6
|
+
HOST = "127.0.0.1"
|
7
|
+
PORT = 25252
|
8
|
+
|
9
|
+
|
10
|
+
def test_server_with_clients():
|
11
|
+
p_server = mp.Process(target=serve, args=(HOST, PORT))
|
12
|
+
p_server.start()
|
13
|
+
|
14
|
+
p_putter = mp.Process(target=dsmq.example_put_client.run, args=(HOST, PORT, 20))
|
15
|
+
p_getter = mp.Process(target=dsmq.example_get_client.run, args=(HOST, PORT, 20))
|
16
|
+
|
17
|
+
p_putter.start()
|
18
|
+
p_getter.start()
|
19
|
+
|
20
|
+
|
21
|
+
if __name__ == "__main__":
|
22
|
+
test_server_with_clients()
|
dsmq/example_get_client.py
CHANGED
dsmq/example_put_client.py
CHANGED
dsmq/{dsmq.py → server.py}
RENAMED
@@ -1,127 +1,74 @@
|
|
1
1
|
import json
|
2
|
-
import
|
2
|
+
import os
|
3
3
|
import sqlite3
|
4
4
|
import sys
|
5
5
|
from threading import Thread
|
6
6
|
import time
|
7
|
+
from websockets.sync.server import serve as ws_serve
|
7
8
|
|
8
9
|
_default_host = "127.0.0.1"
|
9
10
|
_default_port = 30008
|
10
|
-
|
11
|
-
_message_length_offset = 1_000_000
|
12
|
-
_header_length = 23
|
13
11
|
_n_retries = 5
|
14
12
|
_first_retry = 0.01 # seconds
|
13
|
+
_pause = 0.01 # seconds
|
15
14
|
_time_to_live = 600.0 # seconds
|
16
15
|
|
16
|
+
_db_name = "file::memory:?cache=shared"
|
17
|
+
|
18
|
+
# Make this global so it's easy to share
|
19
|
+
dsmq_server = None
|
20
|
+
|
17
21
|
|
18
|
-
def
|
22
|
+
def serve(host=_default_host, port=_default_port):
|
19
23
|
"""
|
20
24
|
For best results, start this running in its own process and walk away.
|
21
25
|
"""
|
22
|
-
|
26
|
+
# Cleanup temp files.
|
27
|
+
# Under some condition
|
28
|
+
# (which I haven't yet been able to pin down)
|
29
|
+
# a file is generated with the db name.
|
30
|
+
# If it is not removed, it gets
|
31
|
+
# treated as a SQLite db on disk,
|
32
|
+
# which dramatically slows it down,
|
33
|
+
# especially the way it's used here for
|
34
|
+
# rapid-fire one-item reads and writes.
|
35
|
+
filenames = os.listdir()
|
36
|
+
for filename in filenames:
|
37
|
+
if filename[: len(_db_name)] == _db_name:
|
38
|
+
os.remove(filename)
|
39
|
+
|
40
|
+
sqlite_conn = sqlite3.connect(_db_name)
|
23
41
|
cursor = sqlite_conn.cursor()
|
24
|
-
|
25
42
|
cursor.execute("""
|
26
43
|
CREATE TABLE IF NOT EXISTS messages (timestamp DOUBLE, topic TEXT, message TEXT)
|
27
44
|
""")
|
28
45
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
s.bind((host, port))
|
37
|
-
s.listen()
|
38
|
-
|
39
|
-
print()
|
40
|
-
print(f"Server started at {host} on port {port}.")
|
41
|
-
print("Waiting for clients...")
|
46
|
+
# Making this global in scope is a way to make it available
|
47
|
+
# to the shutdown operation. It's an awkward construction,
|
48
|
+
# and a method of last resort. (If you stumble across this and
|
49
|
+
# figure out something more elegant, please submit a PR!
|
50
|
+
# or send it to me at brohrer@gmail.com,
|
51
|
+
global dsmq_server
|
42
52
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
53
|
+
dsmq_server = ws_serve(request_handler, host, port)
|
54
|
+
dsmq_server.serve_forever()
|
55
|
+
print()
|
56
|
+
print(f"Server started at {host} on port {port}.")
|
57
|
+
print("Waiting for clients...")
|
47
58
|
|
48
59
|
sqlite_conn.close()
|
49
60
|
|
50
61
|
|
51
|
-
def
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
class DSMQClientSideConnection:
|
56
|
-
def __init__(self, host, port):
|
57
|
-
self.dsmq_conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
58
|
-
self.dsmq_conn.connect((host, port))
|
59
|
-
|
60
|
-
def get(self, topic):
|
61
|
-
msg_dict = {"action": "get", "topic": topic}
|
62
|
-
_send_message(self.dsmq_conn, msg_dict)
|
63
|
-
|
64
|
-
msg = _receive_message(self.dsmq_conn)
|
65
|
-
if msg is None:
|
66
|
-
raise RuntimeError("Connection terminated by server")
|
67
|
-
return msg["message"]
|
68
|
-
|
69
|
-
def put(self, topic, msg_body):
|
70
|
-
msg_dict = {"action": "put", "topic": topic, "message": msg_body}
|
71
|
-
_send_message(self.dsmq_conn, msg_dict)
|
72
|
-
|
73
|
-
|
74
|
-
def _send_message(socket_conn, msg_dict):
|
75
|
-
msg = json.dumps(msg_dict)
|
76
|
-
msg_bytes = bytes(msg, "utf-8")
|
77
|
-
n_bytes = len(msg_bytes)
|
78
|
-
|
79
|
-
# Send a header first.
|
80
|
-
# Add a large number to ensure that the message is the same size each time.
|
81
|
-
header_dict = {"msg_length": n_bytes + _message_length_offset}
|
82
|
-
header = json.dumps(header_dict)
|
83
|
-
header_bytes = bytes(header, "utf-8")
|
84
|
-
socket_conn.sendall(header_bytes)
|
85
|
-
|
86
|
-
socket_conn.sendall(msg_bytes)
|
87
|
-
|
88
|
-
|
89
|
-
def _receive_message(socket_conn):
|
90
|
-
# First receive a header
|
91
|
-
header_data = socket_conn.recv(_header_length)
|
92
|
-
if header_data is None:
|
93
|
-
return None
|
94
|
-
if len(header_data) == 0:
|
95
|
-
return None
|
96
|
-
|
97
|
-
header_str = header_data.decode("utf-8")
|
98
|
-
header = json.loads(header_str)
|
99
|
-
msg_length = header["msg_length"] - _message_length_offset
|
100
|
-
|
101
|
-
data = socket_conn.recv(msg_length)
|
102
|
-
if data is None:
|
103
|
-
return None
|
104
|
-
if len(data) == 0:
|
105
|
-
return None
|
106
|
-
|
107
|
-
msg_str = data.decode("utf-8")
|
108
|
-
msg = json.loads(msg_str)
|
109
|
-
return msg
|
110
|
-
|
111
|
-
|
112
|
-
def _handle_client_connection(socket_conn):
|
113
|
-
sqlite_conn = sqlite3.connect("file:mem1?mode=memory&cache=shared")
|
62
|
+
def request_handler(websocket):
|
63
|
+
sqlite_conn = sqlite3.connect(_db_name)
|
114
64
|
cursor = sqlite_conn.cursor()
|
115
65
|
|
116
66
|
client_creation_time = time.time()
|
117
67
|
last_read_times = {}
|
118
68
|
time_of_last_purge = time.time()
|
119
69
|
|
120
|
-
|
121
|
-
msg =
|
122
|
-
if msg is None:
|
123
|
-
break
|
124
|
-
|
70
|
+
for msg_text in websocket:
|
71
|
+
msg = json.loads(msg_text)
|
125
72
|
topic = msg["topic"]
|
126
73
|
timestamp = time.time()
|
127
74
|
|
@@ -167,7 +114,7 @@ FROM messages,
|
|
167
114
|
SELECT MIN(timestamp) AS min_time
|
168
115
|
FROM messages
|
169
116
|
WHERE topic = :topic
|
170
|
-
|
117
|
+
AND timestamp > :last_read_time
|
171
118
|
) a
|
172
119
|
WHERE topic = :topic
|
173
120
|
AND timestamp = a.min_time
|
@@ -189,9 +136,28 @@ AND timestamp = a.min_time
|
|
189
136
|
# Handle the case where no results are returned
|
190
137
|
message = ""
|
191
138
|
|
192
|
-
|
139
|
+
websocket.send(json.dumps({"message": message}))
|
140
|
+
elif msg["action"] == "shutdown":
|
141
|
+
# Run this from a separate thread to prevent deadlock
|
142
|
+
global dsmq_server
|
143
|
+
print("Shutting down the dsmq server.")
|
144
|
+
|
145
|
+
def shutdown_gracefully(server_to_shutdown):
|
146
|
+
server_to_shutdown.shutdown()
|
147
|
+
time.sleep(_pause)
|
148
|
+
|
149
|
+
filenames = os.listdir()
|
150
|
+
for filename in filenames:
|
151
|
+
if filename[: len(_db_name)] == _db_name:
|
152
|
+
try:
|
153
|
+
os.remove(filename)
|
154
|
+
except FileNotFoundError:
|
155
|
+
pass
|
156
|
+
|
157
|
+
Thread(target=shutdown_gracefully, args=(dsmq_server,)).start()
|
158
|
+
sqlite_conn.close()
|
193
159
|
else:
|
194
|
-
print("Action must either be 'put' or '
|
160
|
+
print("Action must either be 'put', 'get', or 'shudown'")
|
195
161
|
|
196
162
|
# Periodically clean out messages from the queue that are
|
197
163
|
# past their sell buy date.
|
@@ -202,7 +168,7 @@ AND timestamp = a.min_time
|
|
202
168
|
DELETE FROM messages
|
203
169
|
WHERE timestamp < :time_threshold
|
204
170
|
""",
|
205
|
-
{"time_threshold": time_of_last_purge}
|
171
|
+
{"time_threshold": time_of_last_purge},
|
206
172
|
)
|
207
173
|
sqlite_conn.commit()
|
208
174
|
time_of_last_purge = time.time()
|
@@ -214,21 +180,21 @@ if __name__ == "__main__":
|
|
214
180
|
if len(sys.argv) == 3:
|
215
181
|
host = sys.argv[1]
|
216
182
|
port = int(sys.argv[2])
|
217
|
-
|
183
|
+
serve(host=host, port=port)
|
218
184
|
elif len(sys.argv) == 2:
|
219
185
|
host = sys.argv[1]
|
220
|
-
|
186
|
+
serve(host=host)
|
221
187
|
elif len(sys.argv) == 1:
|
222
|
-
|
188
|
+
serve()
|
223
189
|
else:
|
224
190
|
print(
|
225
191
|
"""
|
226
192
|
Try one of these:
|
227
|
-
$ python3
|
193
|
+
$ python3 server.py
|
228
194
|
|
229
|
-
$ python3
|
195
|
+
$ python3 server.py 127.0.0.1
|
230
196
|
|
231
|
-
$ python3
|
197
|
+
$ python3 server.py 127.0.0.1 25853
|
232
198
|
|
233
|
-
|
199
|
+
"""
|
234
200
|
)
|
dsmq/tests/__init__.py
ADDED
File without changes
|
@@ -0,0 +1,238 @@
|
|
1
|
+
import multiprocessing as mp
|
2
|
+
import time
|
3
|
+
from dsmq.server import serve
|
4
|
+
from dsmq.client import connect
|
5
|
+
|
6
|
+
# spawn is the default method on macOS
|
7
|
+
# mp.set_start_method('spawn')
|
8
|
+
|
9
|
+
host = "127.0.0.1"
|
10
|
+
port = 30303
|
11
|
+
|
12
|
+
_short_pause = 0.001
|
13
|
+
_pause = 0.01
|
14
|
+
_long_pause = 0.1
|
15
|
+
_very_long_pause = 0.1
|
16
|
+
|
17
|
+
|
18
|
+
def test_client_server():
|
19
|
+
p_server = mp.Process(target=serve, args=(host, port))
|
20
|
+
p_server.start()
|
21
|
+
mq = connect(host, port)
|
22
|
+
read_completes = False
|
23
|
+
write_completes = False
|
24
|
+
|
25
|
+
n_iter = 11
|
26
|
+
for i in range(n_iter):
|
27
|
+
mq.put("test", f"msg_{i}")
|
28
|
+
write_completes = True
|
29
|
+
|
30
|
+
for i in range(n_iter):
|
31
|
+
msg = mq.get("test")
|
32
|
+
read_completes = True
|
33
|
+
|
34
|
+
assert msg
|
35
|
+
assert write_completes
|
36
|
+
assert read_completes
|
37
|
+
|
38
|
+
mq.shutdown_server()
|
39
|
+
mq.close()
|
40
|
+
|
41
|
+
# It takes a sec to shut down the server
|
42
|
+
time.sleep(_long_pause)
|
43
|
+
assert not p_server.is_alive()
|
44
|
+
|
45
|
+
|
46
|
+
def test_write_one_read_one():
|
47
|
+
p_server = mp.Process(target=serve, args=(host, port))
|
48
|
+
p_server.start()
|
49
|
+
write_client = connect(host, port)
|
50
|
+
read_client = connect(host, port)
|
51
|
+
|
52
|
+
write_client.put("test", "test_msg")
|
53
|
+
|
54
|
+
# It takes a moment for the write to complete
|
55
|
+
time.sleep(_pause)
|
56
|
+
msg = read_client.get("test")
|
57
|
+
|
58
|
+
assert msg == "test_msg"
|
59
|
+
|
60
|
+
write_client.shutdown_server()
|
61
|
+
write_client.close()
|
62
|
+
read_client.close()
|
63
|
+
|
64
|
+
|
65
|
+
def test_get_wait():
|
66
|
+
p_server = mp.Process(target=serve, args=(host, port))
|
67
|
+
p_server.start()
|
68
|
+
write_client = connect(host, port)
|
69
|
+
read_client = connect(host, port)
|
70
|
+
|
71
|
+
write_client.put("test", "test_msg")
|
72
|
+
|
73
|
+
msg = read_client.get_wait("test")
|
74
|
+
|
75
|
+
assert msg == "test_msg"
|
76
|
+
|
77
|
+
write_client.shutdown_server()
|
78
|
+
write_client.close()
|
79
|
+
read_client.close()
|
80
|
+
|
81
|
+
|
82
|
+
def test_multitopics():
|
83
|
+
p_server = mp.Process(target=serve, args=(host, port))
|
84
|
+
p_server.start()
|
85
|
+
write_client = connect(host, port)
|
86
|
+
read_client = connect(host, port)
|
87
|
+
|
88
|
+
write_client.put("test_A", "test_msg_A")
|
89
|
+
write_client.put("test_B", "test_msg_B")
|
90
|
+
|
91
|
+
msg_A = read_client.get_wait("test_A")
|
92
|
+
msg_B = read_client.get_wait("test_B")
|
93
|
+
|
94
|
+
assert msg_A == "test_msg_A"
|
95
|
+
assert msg_B == "test_msg_B"
|
96
|
+
|
97
|
+
write_client.shutdown_server()
|
98
|
+
write_client.close()
|
99
|
+
read_client.close()
|
100
|
+
|
101
|
+
|
102
|
+
def test_client_history_cutoff():
|
103
|
+
p_server = mp.Process(target=serve, args=(host, port))
|
104
|
+
p_server.start()
|
105
|
+
write_client = connect(host, port)
|
106
|
+
write_client.put("test", "test_msg")
|
107
|
+
time.sleep(_pause)
|
108
|
+
|
109
|
+
read_client = connect(host, port)
|
110
|
+
msg = read_client.get("test")
|
111
|
+
|
112
|
+
assert msg == ""
|
113
|
+
|
114
|
+
write_client.shutdown_server()
|
115
|
+
write_client.close()
|
116
|
+
read_client.close()
|
117
|
+
|
118
|
+
|
119
|
+
def test_two_write_clients():
|
120
|
+
p_server = mp.Process(target=serve, args=(host, port))
|
121
|
+
p_server.start()
|
122
|
+
write_client_A = connect(host, port)
|
123
|
+
write_client_B = connect(host, port)
|
124
|
+
read_client = connect(host, port)
|
125
|
+
|
126
|
+
write_client_A.put("test", "test_msg_A")
|
127
|
+
# Wait briefly, to ensure the order of writes
|
128
|
+
time.sleep(_pause)
|
129
|
+
write_client_B.put("test", "test_msg_B")
|
130
|
+
msg_A = read_client.get_wait("test")
|
131
|
+
msg_B = read_client.get_wait("test")
|
132
|
+
|
133
|
+
assert msg_A == "test_msg_A"
|
134
|
+
assert msg_B == "test_msg_B"
|
135
|
+
|
136
|
+
write_client_A.shutdown_server()
|
137
|
+
write_client_A.close()
|
138
|
+
write_client_B.close()
|
139
|
+
read_client.close()
|
140
|
+
|
141
|
+
|
142
|
+
def test_two_read_clients():
|
143
|
+
p_server = mp.Process(target=serve, args=(host, port))
|
144
|
+
p_server.start()
|
145
|
+
write_client = connect(host, port)
|
146
|
+
read_client_A = connect(host, port)
|
147
|
+
read_client_B = connect(host, port)
|
148
|
+
|
149
|
+
write_client.put("test", "test_msg")
|
150
|
+
msg_A = read_client_A.get_wait("test")
|
151
|
+
msg_B = read_client_B.get_wait("test")
|
152
|
+
|
153
|
+
assert msg_A == "test_msg"
|
154
|
+
assert msg_B == "test_msg"
|
155
|
+
|
156
|
+
write_client.shutdown_server()
|
157
|
+
write_client.close()
|
158
|
+
read_client_A.close()
|
159
|
+
read_client_B.close()
|
160
|
+
|
161
|
+
|
162
|
+
def speed_write(stop_flag):
|
163
|
+
fast_write_client = connect(host, port)
|
164
|
+
while True:
|
165
|
+
fast_write_client.put("speed_test", "speed")
|
166
|
+
if stop_flag.is_set():
|
167
|
+
break
|
168
|
+
fast_write_client.close()
|
169
|
+
|
170
|
+
|
171
|
+
def test_speed_writing():
|
172
|
+
p_server = mp.Process(target=serve, args=(host, port))
|
173
|
+
p_server.start()
|
174
|
+
write_client = connect(host, port)
|
175
|
+
read_client = connect(host, port)
|
176
|
+
|
177
|
+
stop_flag = mp.Event()
|
178
|
+
p_speed_write = mp.Process(target=speed_write, args=(stop_flag,))
|
179
|
+
p_speed_write.start()
|
180
|
+
time.sleep(_pause)
|
181
|
+
|
182
|
+
# time_a = time.time()
|
183
|
+
write_client.put("test", "test_msg")
|
184
|
+
# time_b = time.time()
|
185
|
+
msg = read_client.get_wait("test")
|
186
|
+
# time_c = time.time()
|
187
|
+
|
188
|
+
# write_time = int((time_b - time_a) * 1e6)
|
189
|
+
# read_time = int((time_c - time_b) * 1e6)
|
190
|
+
# print(f"write time: {write_time} us, read time: {read_time} us")
|
191
|
+
|
192
|
+
assert msg == "test_msg"
|
193
|
+
|
194
|
+
stop_flag.set()
|
195
|
+
p_speed_write.join()
|
196
|
+
read_client.close()
|
197
|
+
write_client.shutdown_server()
|
198
|
+
write_client.close()
|
199
|
+
|
200
|
+
|
201
|
+
def speed_read(stop_flag):
|
202
|
+
fast_read_client = connect(host, port)
|
203
|
+
while True:
|
204
|
+
fast_read_client.get("speed_test")
|
205
|
+
if stop_flag.is_set():
|
206
|
+
break
|
207
|
+
fast_read_client.close()
|
208
|
+
|
209
|
+
|
210
|
+
def test_speed_reading():
|
211
|
+
p_server = mp.Process(target=serve, args=(host, port))
|
212
|
+
p_server.start()
|
213
|
+
write_client = connect(host, port)
|
214
|
+
read_client = connect(host, port)
|
215
|
+
|
216
|
+
stop_flag = mp.Event()
|
217
|
+
p_speed_read = mp.Process(target=speed_read, args=(stop_flag,))
|
218
|
+
p_speed_read.start()
|
219
|
+
|
220
|
+
# time_a = time.time()
|
221
|
+
write_client.put("test", "test_msg")
|
222
|
+
# time_b = time.time()
|
223
|
+
msg = read_client.get_wait("test")
|
224
|
+
# time_c = time.time()
|
225
|
+
|
226
|
+
# write_time = int((time_b - time_a) * 1e6)
|
227
|
+
# read_time = int((time_c - time_b) * 1e6)
|
228
|
+
# print(f"write time: {write_time} us, read time: {read_time} us")
|
229
|
+
|
230
|
+
assert msg == "test_msg"
|
231
|
+
|
232
|
+
stop_flag.set()
|
233
|
+
p_speed_read.join()
|
234
|
+
p_speed_read.kill()
|
235
|
+
read_client.close()
|
236
|
+
write_client.shutdown_server()
|
237
|
+
# time.sleep(_pause)
|
238
|
+
write_client.close()
|
@@ -1,8 +1,11 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: dsmq
|
3
|
-
Version:
|
3
|
+
Version: 1.1.0
|
4
4
|
Summary: A dead simple message queue
|
5
|
+
License-File: LICENSE
|
5
6
|
Requires-Python: >=3.10
|
7
|
+
Requires-Dist: pytest>=8.3.4
|
8
|
+
Requires-Dist: websockets>=14.1
|
6
9
|
Description-Content-Type: text/markdown
|
7
10
|
|
8
11
|
# Dead Simple Message Queue
|
@@ -26,9 +29,9 @@ pip install dsmq
|
|
26
29
|
As in `src/dsmq/example_server.py`
|
27
30
|
|
28
31
|
```python
|
29
|
-
from dsmq import
|
32
|
+
from dsmq.server import serve
|
30
33
|
|
31
|
-
|
34
|
+
serve(host="127.0.0.1", port=30008)
|
32
35
|
```
|
33
36
|
|
34
37
|
### Connect a client to a dsmq server
|
@@ -36,7 +39,9 @@ dsmq.start_server(host="127.0.0.1", port=30008)
|
|
36
39
|
As in `src/dsmq/example_put_client.py`
|
37
40
|
|
38
41
|
```python
|
39
|
-
|
42
|
+
from dsmq.client import connect
|
43
|
+
|
44
|
+
mq = connect(host="127.0.0.1", port=12345)
|
40
45
|
```
|
41
46
|
### Add a message to a queue
|
42
47
|
|
@@ -66,7 +71,7 @@ managed
|
|
66
71
|
```python
|
67
72
|
import multiprocessing as mp
|
68
73
|
|
69
|
-
p_mq = mp.Process(target=
|
74
|
+
p_mq = mp.Process(target=serve, args=(config.MQ_HOST, config.MQ_PORT))
|
70
75
|
p_mq.start()
|
71
76
|
|
72
77
|
p_mq.join()
|
@@ -78,11 +83,11 @@ p_mq.close()
|
|
78
83
|
### Demo
|
79
84
|
|
80
85
|
1. Open 3 separate terminal windows.
|
81
|
-
1. In the first, run `src/dsmq/
|
86
|
+
1. In the first, run `src/dsmq/server.py` as a script.
|
82
87
|
1. In the second, run `src/dsmq/example_put_client.py`.
|
83
88
|
1. In the third, run `src/dsmq/example_get_client.py`.
|
84
89
|
|
85
|
-
Alternatively,
|
90
|
+
Alternatively, you can run them all at once with `src/dsmq/demo.py`.
|
86
91
|
|
87
92
|
## How it works
|
88
93
|
|
@@ -109,9 +114,9 @@ get larger than your RAM, you will reach an out-of-memory condition.
|
|
109
114
|
|
110
115
|
|
111
116
|
# API Reference
|
112
|
-
[[source](https://github.com/brohrer/dsmq/blob/main/src/dsmq/
|
117
|
+
[[source](https://github.com/brohrer/dsmq/blob/main/src/dsmq/serve.py)]
|
113
118
|
|
114
|
-
### `
|
119
|
+
### `serve(host="127.0.0.1", port=30008)`
|
115
120
|
|
116
121
|
Kicks off the mesage queue server. This process will be the central exchange
|
117
122
|
for all incoming and outgoing messages.
|
@@ -119,9 +124,9 @@ for all incoming and outgoing messages.
|
|
119
124
|
- `port` (int), port. These will be used by all clients.
|
120
125
|
Non-privileged ports are numbered 1024 and higher.
|
121
126
|
|
122
|
-
### `
|
127
|
+
### `connect(host="127.0.0.1", port=30008)`
|
123
128
|
|
124
|
-
Connects to an existing message queue server.
|
129
|
+
Connects a client to an existing message queue server.
|
125
130
|
- `host` (str), IP address of the *server*.
|
126
131
|
- `port` (int), port on which the server is listening.
|
127
132
|
- returns a `DSMQClientSideConnection` object.
|
@@ -146,3 +151,30 @@ connected to the server.
|
|
146
151
|
- returns str, the content of the message. If there was no eligble message
|
147
152
|
in the topic, or the topic doesn't yet exist,
|
148
153
|
returns `""`.
|
154
|
+
|
155
|
+
### `get_wait(topic)`
|
156
|
+
|
157
|
+
A variant of `get()` that retries a few times until it gets
|
158
|
+
a non-empty message. Adjust internal values `_n_tries` and `_initial_retry`
|
159
|
+
to change how persistent it will be.
|
160
|
+
|
161
|
+
- `topic` (str)
|
162
|
+
- returns str, the content of the message. If there was no eligble message
|
163
|
+
in the topic after the allotted number of tries,
|
164
|
+
or the topic doesn't yet exist,
|
165
|
+
returns `""`.
|
166
|
+
|
167
|
+
### `shutdown_server()`
|
168
|
+
|
169
|
+
Gracefully shut down the server, through the client connection.
|
170
|
+
|
171
|
+
### `close()`
|
172
|
+
|
173
|
+
Gracefully shut down the client connection.
|
174
|
+
|
175
|
+
# Testing
|
176
|
+
|
177
|
+
Run all the tests in `src/dsmq/tests/` with pytest, for example
|
178
|
+
```
|
179
|
+
uv run pytest
|
180
|
+
```
|
@@ -0,0 +1,12 @@
|
|
1
|
+
dsmq/__init__.py,sha256=YCgbnQAk8YbtHRyMcU0v2O7RdRhPhlT-vS_q40a7Q6g,50
|
2
|
+
dsmq/client.py,sha256=2i3BhpQM71j-uZlqiJvzQgywi1y07K1VgQe8i2Koi0g,2070
|
3
|
+
dsmq/demo.py,sha256=K53cC5kN7K4kNJlPq7c5OTIMHRCKTo9hYX2aIos57rU,542
|
4
|
+
dsmq/example_get_client.py,sha256=PvAsDGEAH1kVBifLVg2rx8ZxnAZmvzVCvZq13VgpLds,301
|
5
|
+
dsmq/example_put_client.py,sha256=QxDc3i7KAjjhpwxRRpI0Ke5KTNSPuBf9kkcGyTvUEaw,353
|
6
|
+
dsmq/server.py,sha256=iiaa7IoxZy1143vwTO7eMD-kzUKTTVff37E3KPpJ4cU,6010
|
7
|
+
dsmq/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
dsmq/tests/integration_test.py,sha256=TJyx_wsFWeSvYuMF7vCNSGcWstTSLZbuAeKrxrQcj98,5930
|
9
|
+
dsmq-1.1.0.dist-info/METADATA,sha256=S72f4YPsNO07F8W4cnrhGGEW-4Eoao9BBh2VP28uziM,4730
|
10
|
+
dsmq-1.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
11
|
+
dsmq-1.1.0.dist-info/licenses/LICENSE,sha256=3Yu1mAp5VsKmnDtzkiOY7BdmrLeNwwZ3t6iWaLnlL0Y,1071
|
12
|
+
dsmq-1.1.0.dist-info/RECORD,,
|
dsmq/demo_linux.py
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
# Heads up: This script only works on Linux because of use of "fork" method.
|
2
|
-
import multiprocessing as mp
|
3
|
-
import dsmq
|
4
|
-
import example_get_client
|
5
|
-
import example_put_client
|
6
|
-
|
7
|
-
mp.set_start_method("fork")
|
8
|
-
|
9
|
-
HOST = "127.0.0.1"
|
10
|
-
PORT = 25252
|
11
|
-
|
12
|
-
|
13
|
-
def test_server_with_clients():
|
14
|
-
p_server = mp.Process(target=dsmq.start_server, args=(HOST, PORT))
|
15
|
-
p_server.start()
|
16
|
-
|
17
|
-
p_putter = mp.Process(target=example_put_client.run, args=(HOST, PORT, 20))
|
18
|
-
p_getter = mp.Process(target=example_get_client.run, args=(HOST, PORT, 20))
|
19
|
-
|
20
|
-
p_putter.start()
|
21
|
-
p_getter.start()
|
22
|
-
|
23
|
-
|
24
|
-
if __name__ == "__main__":
|
25
|
-
test_server_with_clients()
|
dsmq/example_server.py
DELETED
dsmq-0.7.0.dist-info/RECORD
DELETED
@@ -1,10 +0,0 @@
|
|
1
|
-
dsmq/__init__.py,sha256=YCgbnQAk8YbtHRyMcU0v2O7RdRhPhlT-vS_q40a7Q6g,50
|
2
|
-
dsmq/demo_linux.py,sha256=7yLglGmirDLuuyMxppYSK-dfx2Fg2Q0dIWB4cl2yV1c,622
|
3
|
-
dsmq/dsmq.py,sha256=f-dR-Pjjn_gDx-EK1evYQWvDq8ypXbhx6DaCWHCInSc,6744
|
4
|
-
dsmq/example_get_client.py,sha256=chFfB2949PBENmgdUc3ASrATq1m4wvHGBzEnOC-o_Xs,296
|
5
|
-
dsmq/example_put_client.py,sha256=mUKCRhmUieMZEpHLFWFvzeKB6IR7A8l4tWN8TvPzdKU,348
|
6
|
-
dsmq/example_server.py,sha256=kkXOPaaTzVxf9_iIM76zU9pZhkPna_1vcGWkPrhCjus,61
|
7
|
-
dsmq-0.7.0.dist-info/METADATA,sha256=NFG4GQwX9RqIKLR3xvFelzuTVebieS4sJqZ9uFu24Lw,4006
|
8
|
-
dsmq-0.7.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
|
9
|
-
dsmq-0.7.0.dist-info/licenses/LICENSE,sha256=3Yu1mAp5VsKmnDtzkiOY7BdmrLeNwwZ3t6iWaLnlL0Y,1071
|
10
|
-
dsmq-0.7.0.dist-info/RECORD,,
|
File without changes
|