django-sockets 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_sockets/__init__.py +0 -0
- django_sockets/broadcaster.py +99 -0
- django_sockets/middleware.py +56 -0
- django_sockets/pubsub.py +315 -0
- django_sockets/sockets.py +206 -0
- django_sockets/utils.py +215 -0
- django_sockets-1.0.0.dist-info/LICENSE +21 -0
- django_sockets-1.0.0.dist-info/METADATA +191 -0
- django_sockets-1.0.0.dist-info/RECORD +11 -0
- django_sockets-1.0.0.dist-info/WHEEL +5 -0
- django_sockets-1.0.0.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import asyncio, logging
|
|
2
|
+
|
|
3
|
+
from .pubsub import PubSubLayer
|
|
4
|
+
from .utils import ensure_loop_running, get_config
|
|
5
|
+
|
|
6
|
+
logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Broadcaster:
|
|
10
|
+
def __init__(self, *args, loop=None, config=None, **kwargs):
|
|
11
|
+
self.__loop__ = ensure_loop_running(loop)
|
|
12
|
+
self.pubsub_layer = self.__get_pubsub_layer__(config=config)
|
|
13
|
+
self.__usable__ = self.pubsub_layer is not None
|
|
14
|
+
|
|
15
|
+
# Sync Functions
|
|
16
|
+
def broadcast(self, channel: str, data: [dict | list]):
|
|
17
|
+
"""
|
|
18
|
+
Broadcast data to a specific channel or all channels that this socket server is subscribed to.
|
|
19
|
+
|
|
20
|
+
Requires:
|
|
21
|
+
|
|
22
|
+
- channel: str = The channel to broadcast the data to
|
|
23
|
+
- data: [dict|list] = The data to broadcast to the channel
|
|
24
|
+
- Note: This data must be JSON serializable
|
|
25
|
+
"""
|
|
26
|
+
asyncio.run_coroutine_threadsafe(
|
|
27
|
+
self.async_broadcast(channel, data), self.__loop__
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
def subscribe(self, channel: str):
|
|
31
|
+
"""
|
|
32
|
+
Subscribe to a channel to receive data from broadcasts
|
|
33
|
+
|
|
34
|
+
Requires:
|
|
35
|
+
|
|
36
|
+
- channel: str = The channel to subscribe to
|
|
37
|
+
"""
|
|
38
|
+
asyncio.run_coroutine_threadsafe(
|
|
39
|
+
self.async_subscribe(channel), self.__loop__
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
# Async Functions
|
|
43
|
+
async def async_broadcast(self, channel: str, data):
|
|
44
|
+
"""
|
|
45
|
+
Broadcast data to a channel where all relevant clients will receive the data
|
|
46
|
+
and send it to the client
|
|
47
|
+
|
|
48
|
+
Requires:
|
|
49
|
+
|
|
50
|
+
- channel: str = The channel to broadcast the data to
|
|
51
|
+
- data: [dict|list] = The data to broadcast to the channel
|
|
52
|
+
- Note: This data must be JSON serializable
|
|
53
|
+
"""
|
|
54
|
+
self.__warn_on_not_usable__()
|
|
55
|
+
await self.pubsub_layer.send(str(channel), data)
|
|
56
|
+
|
|
57
|
+
async def async_subscribe(self, channel: str):
|
|
58
|
+
"""
|
|
59
|
+
Subscribe to a channel to receive data from broadcasts
|
|
60
|
+
|
|
61
|
+
Requires:
|
|
62
|
+
|
|
63
|
+
- channel: str = The channel to subscribe to
|
|
64
|
+
"""
|
|
65
|
+
self.__warn_on_not_usable__()
|
|
66
|
+
await self.pubsub_layer.subscribe(str(channel))
|
|
67
|
+
|
|
68
|
+
async def async_receive_broadcast(self):
|
|
69
|
+
"""
|
|
70
|
+
Receive a broadcast from a channel that this socket server is subscribed to
|
|
71
|
+
|
|
72
|
+
Returns:
|
|
73
|
+
|
|
74
|
+
- channel: str = The channel that the data was broadcasted to
|
|
75
|
+
- data: [dict|list] = The data that was broadcasted to the channel
|
|
76
|
+
- Note: This data must be JSON serializable
|
|
77
|
+
"""
|
|
78
|
+
data = await self.pubsub_layer.receive()
|
|
79
|
+
return data["channel"], data["data"]
|
|
80
|
+
|
|
81
|
+
# Internal Methods
|
|
82
|
+
def __get_pubsub_layer__(self, config=None):
|
|
83
|
+
"""
|
|
84
|
+
A method to get the PubSubLayer given the settings.DJANGO_SOCKETS_CONFIG
|
|
85
|
+
"""
|
|
86
|
+
config = get_config(config)
|
|
87
|
+
if "hosts" in config:
|
|
88
|
+
return PubSubLayer(**config)
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
def __warn_on_not_usable__(self):
|
|
92
|
+
"""
|
|
93
|
+
Warn the user if the broadcaster is not usable
|
|
94
|
+
"""
|
|
95
|
+
if not self.__usable__:
|
|
96
|
+
logger.log(
|
|
97
|
+
logging.ERROR,
|
|
98
|
+
"No hosts provided in settings.DJANGO_SOCKETS_CONFIG. Broadcasting / Subscribing is not possible.",
|
|
99
|
+
)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from .utils import database_sync_to_async
|
|
2
|
+
|
|
3
|
+
def get_anonymous_user_obj():
|
|
4
|
+
try:
|
|
5
|
+
return AnonymousUser()
|
|
6
|
+
except:
|
|
7
|
+
pass
|
|
8
|
+
try:
|
|
9
|
+
from django.contrib.auth.models import AnonymousUser
|
|
10
|
+
return AnonymousUser()
|
|
11
|
+
except:
|
|
12
|
+
pass
|
|
13
|
+
return None
|
|
14
|
+
|
|
15
|
+
def get_drf_token_obj():
|
|
16
|
+
try:
|
|
17
|
+
return Token
|
|
18
|
+
except:
|
|
19
|
+
pass
|
|
20
|
+
try:
|
|
21
|
+
from rest_framework.authtoken.models import Token
|
|
22
|
+
return Token
|
|
23
|
+
except:
|
|
24
|
+
pass
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
@database_sync_to_async
|
|
28
|
+
def get_drf_user(user_token):
|
|
29
|
+
if user_token is not None:
|
|
30
|
+
try:
|
|
31
|
+
token = get_drf_token_obj().objects.get(key=user_token)
|
|
32
|
+
return token.user
|
|
33
|
+
except:
|
|
34
|
+
pass
|
|
35
|
+
return get_anonymous_user_obj()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def get_query_arg(query_string, arg, default=None):
|
|
39
|
+
try:
|
|
40
|
+
return (
|
|
41
|
+
dict((x.split("=") for x in query_string.decode().split("&")))
|
|
42
|
+
).get(arg, default)
|
|
43
|
+
except:
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class DRFTokenAuthMiddleware:
|
|
48
|
+
def __init__(self, app):
|
|
49
|
+
self.app = app
|
|
50
|
+
|
|
51
|
+
async def __call__(self, scope, receive, send):
|
|
52
|
+
scope = dict(scope)
|
|
53
|
+
scope["user"] = await get_drf_user(
|
|
54
|
+
get_query_arg(scope["query_string"], "user_token")
|
|
55
|
+
)
|
|
56
|
+
return await self.app(scope, receive, send)
|
django_sockets/pubsub.py
ADDED
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
import asyncio, logging, binascii, msgpack, logging, uuid
|
|
2
|
+
from redis.asyncio import Redis, ConnectionPool, sentinel
|
|
3
|
+
|
|
4
|
+
logger = logging.getLogger(__name__)
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class PubSubLayer:
|
|
8
|
+
"""
|
|
9
|
+
A meta PubSub Layer that uses Redis's pub/sub functionality and allows multiple subscriptions to
|
|
10
|
+
different channels that may or may not be on different cache shards.
|
|
11
|
+
|
|
12
|
+
This layer is designed to be used with asyncio and is not necessarily thread-safe.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(
|
|
16
|
+
self,
|
|
17
|
+
hosts=None,
|
|
18
|
+
):
|
|
19
|
+
"""
|
|
20
|
+
Initialize the PubSub Layer
|
|
21
|
+
|
|
22
|
+
Requires:
|
|
23
|
+
- hosts: A list of dictionaries
|
|
24
|
+
- Note: This uses a redis connection pool or a sentinel connection pool to connect to the Redis server.
|
|
25
|
+
- Each dictionary should contain the following key value pairs (depending on the connection pool type):
|
|
26
|
+
- ConnectionPool:
|
|
27
|
+
- address: The address of the Redis server
|
|
28
|
+
- EG: 'redis://0.0.0.0:6379'
|
|
29
|
+
- Note: This can be used alone or in conjunction with other connection pool keys
|
|
30
|
+
- host: The host of the Redis server
|
|
31
|
+
- Note: This must be used with the port key and can be used in conjunction with the other connection pool keys
|
|
32
|
+
- port: The port of the Redis server
|
|
33
|
+
- Note: This must be used with the host key and can be used in conjunction with the other connection pool keys
|
|
34
|
+
- Other keys listed in the redis-py ConnectionPool documentation
|
|
35
|
+
- SentinelConnectionPool:
|
|
36
|
+
- master_name: The name of the master in a Redis Sentinel setup
|
|
37
|
+
- sentinels: A list of sentinel addresses
|
|
38
|
+
- sentinel_kwargs: A dictionary of keyword arguments to pass to the sentinel connect
|
|
39
|
+
- Other keys listed in the redis-py SentinelConnectionPool documentation
|
|
40
|
+
"""
|
|
41
|
+
self.queue = asyncio.Queue()
|
|
42
|
+
self.subscriptions = dict()
|
|
43
|
+
if not isinstance(hosts, list):
|
|
44
|
+
raise ValueError("Hosts must be a list of dictionaries")
|
|
45
|
+
if len(hosts) == 0:
|
|
46
|
+
raise ValueError("Hosts must contain at least one dictionary")
|
|
47
|
+
self.shards = [ShardConnection(host, self) for host in hosts]
|
|
48
|
+
|
|
49
|
+
# PubSub methods
|
|
50
|
+
async def subscribe(self, channel: str):
|
|
51
|
+
"""
|
|
52
|
+
Subscribe to a channel
|
|
53
|
+
"""
|
|
54
|
+
shard = self.__get_shard__(channel)
|
|
55
|
+
if channel not in self.subscriptions:
|
|
56
|
+
self.subscriptions[channel] = shard
|
|
57
|
+
await shard.subscribe(channel)
|
|
58
|
+
|
|
59
|
+
async def unsubscribe(self, channel: str):
|
|
60
|
+
"""
|
|
61
|
+
Unsubscribe from a channel
|
|
62
|
+
"""
|
|
63
|
+
if channel in self.subscriptions:
|
|
64
|
+
shard = self.subscriptions.pop(channel)
|
|
65
|
+
await shard.unsubscribe(channel)
|
|
66
|
+
|
|
67
|
+
async def send(self, channel: str, data):
|
|
68
|
+
"""
|
|
69
|
+
Send data to a channel
|
|
70
|
+
"""
|
|
71
|
+
shard = self.__get_shard__(channel)
|
|
72
|
+
await shard.publish(channel, data)
|
|
73
|
+
|
|
74
|
+
async def receive(self) -> dict | None:
|
|
75
|
+
"""
|
|
76
|
+
Get the next item from the queue. This will hang until an item is available.
|
|
77
|
+
|
|
78
|
+
If the queue has been closed, cleanup and raise an exception to exit the calling task.
|
|
79
|
+
|
|
80
|
+
Format:
|
|
81
|
+
{
|
|
82
|
+
'channel': str,
|
|
83
|
+
'data': Any
|
|
84
|
+
}
|
|
85
|
+
"""
|
|
86
|
+
try:
|
|
87
|
+
return await self.queue.get()
|
|
88
|
+
except (asyncio.CancelledError, asyncio.TimeoutError, GeneratorExit):
|
|
89
|
+
# Cleanup / unsubscribe on interruptions / exits / timeouts
|
|
90
|
+
await self.flush()
|
|
91
|
+
# Raise an exception to exit the calling task
|
|
92
|
+
raise asyncio.CancelledError
|
|
93
|
+
|
|
94
|
+
async def flush(self):
|
|
95
|
+
"""
|
|
96
|
+
Flush the layer and close all connections.
|
|
97
|
+
"""
|
|
98
|
+
for shard in set(self.subscriptions.values()):
|
|
99
|
+
try:
|
|
100
|
+
await shard.flush()
|
|
101
|
+
except asyncio.CancelledError:
|
|
102
|
+
raise asyncio.CancelledError
|
|
103
|
+
except BaseException as e:
|
|
104
|
+
logger.exception(
|
|
105
|
+
f"Exception while flushing shard connection: {e}"
|
|
106
|
+
)
|
|
107
|
+
self.subscriptions = dict()
|
|
108
|
+
|
|
109
|
+
# Utility Methods
|
|
110
|
+
def __get_shard__(self, channel):
|
|
111
|
+
"""
|
|
112
|
+
Return the shard that is used for this channel.
|
|
113
|
+
|
|
114
|
+
This is done by assigning a shard index location based on the CRC32 of the channel name.
|
|
115
|
+
"""
|
|
116
|
+
if len(self.shards) == 1:
|
|
117
|
+
shard_index = 0
|
|
118
|
+
else:
|
|
119
|
+
hash_val = binascii.crc32(channel.encode("utf8")) & 0xFFF
|
|
120
|
+
shard_index = int(hash_val / (4096 / float(len(self.shards))))
|
|
121
|
+
return self.shards[shard_index]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class ShardConnection:
|
|
125
|
+
def __init__(self, host, pubsub_layer_obj, prefix="pubsub"):
|
|
126
|
+
self.connection_pool = self.__get_connection_pool__(host)
|
|
127
|
+
self.pubsub_layer_obj = pubsub_layer_obj
|
|
128
|
+
self.lock = asyncio.Lock()
|
|
129
|
+
self.connection = None
|
|
130
|
+
self.pubsub = None
|
|
131
|
+
self.receiver_task = None
|
|
132
|
+
self.subscriptions = set()
|
|
133
|
+
self.prefix = prefix
|
|
134
|
+
|
|
135
|
+
# PubSub methods
|
|
136
|
+
async def subscribe(self, channel):
|
|
137
|
+
channel = self.__get_channel_name__(channel)
|
|
138
|
+
async with self.lock:
|
|
139
|
+
if channel in self.subscriptions:
|
|
140
|
+
return
|
|
141
|
+
await self.__ensure_connection__()
|
|
142
|
+
await self.pubsub.subscribe(channel)
|
|
143
|
+
self.subscriptions.add(channel)
|
|
144
|
+
# Drop out of the lock to start the receiver task which requires the lock to be released
|
|
145
|
+
await self.ensure_receiver_task()
|
|
146
|
+
|
|
147
|
+
async def unsubscribe(self, channel):
|
|
148
|
+
channel = self.__get_channel_name__(channel)
|
|
149
|
+
async with self.lock:
|
|
150
|
+
if channel not in self.subscriptions:
|
|
151
|
+
return
|
|
152
|
+
await self.__ensure_connection__()
|
|
153
|
+
await self.pubsub.unsubscribe(channel)
|
|
154
|
+
self.subscriptions.remove(channel)
|
|
155
|
+
if len(self.subscriptions) == 0:
|
|
156
|
+
await self.flush()
|
|
157
|
+
|
|
158
|
+
async def publish(self, channel, message):
|
|
159
|
+
channel = self.__get_channel_name__(channel)
|
|
160
|
+
async with self.lock:
|
|
161
|
+
await self.__ensure_connection__()
|
|
162
|
+
message = self.__serialize__(message)
|
|
163
|
+
# if the message is larger than 1MB, then save it as a uuid in the same cache and send the uuid
|
|
164
|
+
# This helps bypass the 32 MB limit on pubsub queue size for most cache servers
|
|
165
|
+
# Ensure that this objeect times out after 60s to keep the cache clean
|
|
166
|
+
if len(message) > 1024 * 1024:
|
|
167
|
+
msg_loc_key = f"{self.prefix}.{str(uuid.uuid4())}"
|
|
168
|
+
await self.connection.set(msg_loc_key, message, ex=60)
|
|
169
|
+
message = self.__serialize__(f"msg:{msg_loc_key}")
|
|
170
|
+
await self.connection.publish(channel, message)
|
|
171
|
+
|
|
172
|
+
async def ensure_receiver_task(self):
|
|
173
|
+
async with self.lock:
|
|
174
|
+
if self.receiver_task is None:
|
|
175
|
+
self.receiver_task = asyncio.create_task(
|
|
176
|
+
self.__receiver_task__()
|
|
177
|
+
)
|
|
178
|
+
# This is needed to continue the main coroutine execution after create_task
|
|
179
|
+
await asyncio.sleep(0)
|
|
180
|
+
|
|
181
|
+
async def flush(self):
|
|
182
|
+
# Flushing is not locked since it can be called from inside the lock
|
|
183
|
+
if self.receiver_task:
|
|
184
|
+
self.receiver_task.cancel()
|
|
185
|
+
try:
|
|
186
|
+
await self.receiver_task
|
|
187
|
+
except asyncio.CancelledError:
|
|
188
|
+
pass
|
|
189
|
+
self.receiver_task = None
|
|
190
|
+
if self.pubsub:
|
|
191
|
+
await self.pubsub.close()
|
|
192
|
+
self.pubsub = None
|
|
193
|
+
if self.connection:
|
|
194
|
+
await self.connection.close()
|
|
195
|
+
self.connection = None
|
|
196
|
+
|
|
197
|
+
# Tasks
|
|
198
|
+
async def __receiver_task__(self):
|
|
199
|
+
"""
|
|
200
|
+
Start a task to receive messages from the pubsub and put them in the queue
|
|
201
|
+
|
|
202
|
+
This task will run until all subscriptions are removed.
|
|
203
|
+
|
|
204
|
+
It will loop continuously as awaiting the pubsub.get_message will not hang the event loop.
|
|
205
|
+
"""
|
|
206
|
+
# print("RECEIVER TASK STARTING")
|
|
207
|
+
while len(self.subscriptions) > 0:
|
|
208
|
+
try:
|
|
209
|
+
# Make sure pubsub is active and subscribed otherwise wait for subscription to be established
|
|
210
|
+
if self.pubsub and self.pubsub.subscribed:
|
|
211
|
+
# Get messages from the pubsub
|
|
212
|
+
message = await self.pubsub.get_message(
|
|
213
|
+
ignore_subscribe_messages=True
|
|
214
|
+
)
|
|
215
|
+
# If message is not None, put it in the channel queue
|
|
216
|
+
if message:
|
|
217
|
+
message_data = self.__deserialize__(message["data"])
|
|
218
|
+
# If the message was too large, then get that message from the cache
|
|
219
|
+
if isinstance(message_data, str):
|
|
220
|
+
if message_data.startswith("msg:"):
|
|
221
|
+
msg_loc_key = message_data[4:]
|
|
222
|
+
message_data = self.__deserialize__(
|
|
223
|
+
await self.connection.get(msg_loc_key)
|
|
224
|
+
)
|
|
225
|
+
self.pubsub_layer_obj.queue.put_nowait(
|
|
226
|
+
{
|
|
227
|
+
"channel": self.__get_channel_from_name__(
|
|
228
|
+
message["channel"].decode()
|
|
229
|
+
),
|
|
230
|
+
"data": message_data,
|
|
231
|
+
}
|
|
232
|
+
)
|
|
233
|
+
# Wait for a short time to prevent busy waiting
|
|
234
|
+
# This also serves to wait for the pubsub layer to be subscribed to the channel
|
|
235
|
+
await asyncio.sleep(0.1)
|
|
236
|
+
# Exit on cancellation, timeout, or generator exit (for cleanup afer connection is closed)
|
|
237
|
+
except (
|
|
238
|
+
asyncio.CancelledError,
|
|
239
|
+
asyncio.TimeoutError,
|
|
240
|
+
GeneratorExit,
|
|
241
|
+
):
|
|
242
|
+
# print("RECEIVER TASK KILLED")
|
|
243
|
+
raise asyncio.CancelledError
|
|
244
|
+
except:
|
|
245
|
+
logger.exception(
|
|
246
|
+
"Exception while receiving message from pubsub"
|
|
247
|
+
)
|
|
248
|
+
await self.flush()
|
|
249
|
+
|
|
250
|
+
# Utility Methods
|
|
251
|
+
async def __ensure_connection__(self):
|
|
252
|
+
"""
|
|
253
|
+
Ensure that the connection to the cache is established.
|
|
254
|
+
|
|
255
|
+
Note: This should only be called within a lock.
|
|
256
|
+
"""
|
|
257
|
+
if not self.connection:
|
|
258
|
+
self.connection = Redis(connection_pool=self.connection_pool)
|
|
259
|
+
# If the connection failed, then raise an exception
|
|
260
|
+
try:
|
|
261
|
+
is_connected = await self.connection.ping()
|
|
262
|
+
except:
|
|
263
|
+
is_connected = False
|
|
264
|
+
if not is_connected:
|
|
265
|
+
logger.log(
|
|
266
|
+
logging.ERROR,
|
|
267
|
+
"Could not connect to the cache server. Please check your config and make sure that the cache server is running.",
|
|
268
|
+
)
|
|
269
|
+
raise ConnectionError("Could not connect to the cache server.")
|
|
270
|
+
self.pubsub = self.connection.pubsub()
|
|
271
|
+
|
|
272
|
+
def __get_channel_name__(self, channel):
|
|
273
|
+
"""
|
|
274
|
+
Get the channel name with the prefix.
|
|
275
|
+
"""
|
|
276
|
+
return f"{self.prefix}.{channel}"
|
|
277
|
+
|
|
278
|
+
def __get_channel_from_name__(self, channel_name):
|
|
279
|
+
"""
|
|
280
|
+
Get the channel name without the prefix.
|
|
281
|
+
"""
|
|
282
|
+
return channel_name[len(self.prefix) + 1 :]
|
|
283
|
+
|
|
284
|
+
def __serialize__(self, message):
|
|
285
|
+
"""
|
|
286
|
+
Serialize a message into bytes.
|
|
287
|
+
"""
|
|
288
|
+
return msgpack.packb(message)
|
|
289
|
+
|
|
290
|
+
def __deserialize__(self, message):
|
|
291
|
+
"""
|
|
292
|
+
Deserialize a message from bytes.
|
|
293
|
+
"""
|
|
294
|
+
return msgpack.unpackb(message, strict_map_key=False)
|
|
295
|
+
|
|
296
|
+
@staticmethod
|
|
297
|
+
def __get_connection_pool__(host: dict):
|
|
298
|
+
"""
|
|
299
|
+
Get a connection pool from a host dictionary
|
|
300
|
+
"""
|
|
301
|
+
host = host.copy()
|
|
302
|
+
if "address" in host:
|
|
303
|
+
address = host.pop("address")
|
|
304
|
+
return ConnectionPool.from_url(address, **host)
|
|
305
|
+
|
|
306
|
+
master_name = host.pop("master_name", None)
|
|
307
|
+
if master_name is not None:
|
|
308
|
+
sentinels = host.pop("sentinels")
|
|
309
|
+
sentinel_kwargs = host.pop("sentinel_kwargs", None)
|
|
310
|
+
return sentinel.SentinelConnectionPool(
|
|
311
|
+
master_name,
|
|
312
|
+
sentinel.Sentinel(sentinels, sentinel_kwargs=sentinel_kwargs),
|
|
313
|
+
**host,
|
|
314
|
+
)
|
|
315
|
+
return ConnectionPool(**host)
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import json, asyncio, logging
|
|
2
|
+
from .broadcaster import Broadcaster
|
|
3
|
+
from .utils import run_in_thread
|
|
4
|
+
|
|
5
|
+
logger = logging.getLogger(__name__)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BaseSocketServer(Broadcaster):
|
|
9
|
+
def __init__(self, scope, receive, send, config=None):
|
|
10
|
+
"""
|
|
11
|
+
Initialize the socket server
|
|
12
|
+
|
|
13
|
+
Required:
|
|
14
|
+
|
|
15
|
+
- scope: dict = The scope of the websocket connection
|
|
16
|
+
- receive: method = The `get` method for an asyncio.Queue() object that will be used to receive data from the websocket client
|
|
17
|
+
- send: async callable = The function that should send the data to the websocket client
|
|
18
|
+
- Note: This function takes in a dictionary with the following keys:
|
|
19
|
+
- type: str = The type of message to send (always 'websocket.send')
|
|
20
|
+
- text: str = The data to send to the client (the json serialized version of the data sent / broadcasted)
|
|
21
|
+
|
|
22
|
+
Optional:
|
|
23
|
+
|
|
24
|
+
- config: dict = The configuration for the socket server
|
|
25
|
+
- hosts: list = A list of dictionaries that contain the host information for the socket server
|
|
26
|
+
- See: django_sockets.pubsub.PubSubLayer docs for more more comprehensive docs on the config parameter
|
|
27
|
+
- Default:
|
|
28
|
+
- If provided in django settings: settings.DJANGO_SOCKETS_CONFIG
|
|
29
|
+
- Else: {'hosts': [{'address': 'redis://0.0.0.0:6379'}]}
|
|
30
|
+
"""
|
|
31
|
+
self.scope = scope
|
|
32
|
+
self.__receive__ = receive
|
|
33
|
+
self.__send__ = send
|
|
34
|
+
self.is_alive = True
|
|
35
|
+
super().__init__(config=config)
|
|
36
|
+
|
|
37
|
+
# Sync Functions
|
|
38
|
+
def send(self, data: [dict | list | str | float | int]):
|
|
39
|
+
"""
|
|
40
|
+
Send data to the websocket client.
|
|
41
|
+
- Note: This only sends data to the client from which the calling function was called
|
|
42
|
+
- Note: To send data to all clients that are subscribed to a channel, use the broadcast method
|
|
43
|
+
which is inherited from the Broadcaster class
|
|
44
|
+
|
|
45
|
+
Requires:
|
|
46
|
+
|
|
47
|
+
- data: [dict|list|str|float|int] = The data to send to the client
|
|
48
|
+
- Note: This data must be JSON serializable
|
|
49
|
+
"""
|
|
50
|
+
self.run_async(self.async_send(data))
|
|
51
|
+
|
|
52
|
+
def run_async(self, func):
|
|
53
|
+
"""
|
|
54
|
+
Run an async function in the current object's event loop
|
|
55
|
+
"""
|
|
56
|
+
asyncio.run_coroutine_threadsafe(func, self.__loop__)
|
|
57
|
+
|
|
58
|
+
# Async Functions
|
|
59
|
+
async def async_send(self, data: [dict | list | str | float | int]):
|
|
60
|
+
"""
|
|
61
|
+
Send data to the websocket client.
|
|
62
|
+
- Note: To send data to all clients that are subscribed to a channel, use the broadcast method
|
|
63
|
+
which is inherited from the Broadcaster class
|
|
64
|
+
|
|
65
|
+
Requires:
|
|
66
|
+
|
|
67
|
+
- data: [dict|list|str|float|int] = The data to send to the client
|
|
68
|
+
- Note: This data must be JSON serializable
|
|
69
|
+
"""
|
|
70
|
+
if self.__send__ is None:
|
|
71
|
+
logger.log(
|
|
72
|
+
logging.ERROR,
|
|
73
|
+
"The send and async_send functions are not available because the send parameter was not provided when the socket server was initialized. To silence this warning, you can provide a function that simulates some sending behavior.",
|
|
74
|
+
)
|
|
75
|
+
else:
|
|
76
|
+
try:
|
|
77
|
+
json_data = json.dumps(data)
|
|
78
|
+
except:
|
|
79
|
+
raise ValueError("Data must be JSON serializable")
|
|
80
|
+
await self.__send__({"type": "websocket.send", "text": json_data})
|
|
81
|
+
|
|
82
|
+
async def async_handle_received_broadcast(
|
|
83
|
+
self, channel: str, data: [dict | list | str | float | int]
|
|
84
|
+
):
|
|
85
|
+
"""
|
|
86
|
+
Handle a received broadcast from a subscribed channel
|
|
87
|
+
|
|
88
|
+
This method is provided so that it can be overwritten by the user if they want to handle
|
|
89
|
+
received broadcasts from subscriptions in a specific way.
|
|
90
|
+
|
|
91
|
+
By default, this method will send the data to the client using the async_send method
|
|
92
|
+
|
|
93
|
+
In general, this method should only be called by the __broadcast_listener_task__ method
|
|
94
|
+
|
|
95
|
+
Requires:
|
|
96
|
+
|
|
97
|
+
- channel: str = The channel that the data was broadcasted to
|
|
98
|
+
- data: [dict|list|str|float|int] = The data that was broadcasted
|
|
99
|
+
- Note: This data must be JSON serializable
|
|
100
|
+
"""
|
|
101
|
+
await self.async_send(data)
|
|
102
|
+
|
|
103
|
+
# Tasks
|
|
104
|
+
async def __ws_listener_task__(self):
|
|
105
|
+
"""
|
|
106
|
+
Listen for incoming WS data and handle it accordingly
|
|
107
|
+
"""
|
|
108
|
+
if self.__receive__ is None:
|
|
109
|
+
logger.log(
|
|
110
|
+
logging.ERROR,
|
|
111
|
+
"The websocket listener task is not available because the receive parameter was not provided when the socket server was initialized. To silence this warning, you can provide an asyncio.Queue() receive parameter and put items in it to simulate received ws messages.",
|
|
112
|
+
)
|
|
113
|
+
else:
|
|
114
|
+
while self.is_alive:
|
|
115
|
+
data = await self.__receive__()
|
|
116
|
+
if data["type"] == "websocket.receive":
|
|
117
|
+
try:
|
|
118
|
+
data_text = json.loads(data["text"])
|
|
119
|
+
run_in_thread(self.receive, data_text)
|
|
120
|
+
except:
|
|
121
|
+
logger.exception("Invalid JSON data received")
|
|
122
|
+
elif data["type"] == "websocket.disconnect":
|
|
123
|
+
self.__kill__()
|
|
124
|
+
elif data["type"] == "websocket.connect":
|
|
125
|
+
await self.__send__({"type": "websocket.accept"})
|
|
126
|
+
self.connect()
|
|
127
|
+
else:
|
|
128
|
+
raise ValueError(f"Invalid WS data type: {data['type']}")
|
|
129
|
+
|
|
130
|
+
async def __broadcast_listener_task__(self):
|
|
131
|
+
"""
|
|
132
|
+
Handle all messages that were broadcast to subscribed channels
|
|
133
|
+
"""
|
|
134
|
+
# Only handle broadcasts if the broadcaster is usable
|
|
135
|
+
if self.__usable__:
|
|
136
|
+
while self.is_alive:
|
|
137
|
+
try:
|
|
138
|
+
channel, data = await self.async_receive_broadcast()
|
|
139
|
+
await self.async_handle_received_broadcast(channel, data)
|
|
140
|
+
# Cleanup on exit
|
|
141
|
+
except asyncio.CancelledError:
|
|
142
|
+
raise asyncio.CancelledError
|
|
143
|
+
except Exception as e:
|
|
144
|
+
raise e
|
|
145
|
+
|
|
146
|
+
# Lifecycle Methods
|
|
147
|
+
def __kill__(self):
|
|
148
|
+
"""
|
|
149
|
+
Kill the socket server and stop all tasks
|
|
150
|
+
"""
|
|
151
|
+
self.is_alive = False
|
|
152
|
+
|
|
153
|
+
async def async_start_listeners(self):
|
|
154
|
+
try:
|
|
155
|
+
# Create Tasks for the listener and queue processor
|
|
156
|
+
ws_listener_task = asyncio.create_task(self.__ws_listener_task__())
|
|
157
|
+
broadcast_listener_task = asyncio.create_task(
|
|
158
|
+
self.__broadcast_listener_task__()
|
|
159
|
+
)
|
|
160
|
+
# wait until the socket server is killed or the tasks are cancelled
|
|
161
|
+
while self.is_alive:
|
|
162
|
+
await asyncio.sleep(0.2)
|
|
163
|
+
# Catch exits handled by Daphne and allow the tasks to be cancelled
|
|
164
|
+
except asyncio.CancelledError:
|
|
165
|
+
pass
|
|
166
|
+
# Ensure all tasks are cancelled
|
|
167
|
+
ws_listener_task.cancel()
|
|
168
|
+
broadcast_listener_task.cancel()
|
|
169
|
+
# Allow the cancelation to run in the background
|
|
170
|
+
# The next line allows the async function to return
|
|
171
|
+
# and the above tasks to be cancelled in the background
|
|
172
|
+
await asyncio.sleep(0)
|
|
173
|
+
|
|
174
|
+
def start_listeners(self):
|
|
175
|
+
"""
|
|
176
|
+
Start the listeners for the socket server
|
|
177
|
+
"""
|
|
178
|
+
self.run_async(self.async_start_listeners())
|
|
179
|
+
|
|
180
|
+
# Utility Methods
|
|
181
|
+
@classmethod
|
|
182
|
+
async def as_asgi(cls, scope, receive, send):
|
|
183
|
+
"""
|
|
184
|
+
An ASGI application runner function that can be called by Daphne.
|
|
185
|
+
|
|
186
|
+
This creates a new socket server instance and starts the listeners in the background.
|
|
187
|
+
"""
|
|
188
|
+
# Wrap in a try / except block to catch unclean exits handled by Daphne
|
|
189
|
+
socket_server = cls(scope, receive, send)
|
|
190
|
+
await socket_server.async_start_listeners()
|
|
191
|
+
|
|
192
|
+
# Placeholder Methods
|
|
193
|
+
def receive(self, data):
|
|
194
|
+
"""
|
|
195
|
+
Placeholder method for the receive method that must be overwritten by the user
|
|
196
|
+
|
|
197
|
+
This is the method that will be called when data is received from the ws client.
|
|
198
|
+
"""
|
|
199
|
+
raise NotImplementedError(
|
|
200
|
+
"The receive method must be implemented by the user"
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
def connect(self):
|
|
204
|
+
"""
|
|
205
|
+
Placeholder method for the connect method that can be overwritten by the user.
|
|
206
|
+
"""
|
django_sockets/utils.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
from django.db import close_old_connections
|
|
2
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
3
|
+
from django.urls.exceptions import Resolver404
|
|
4
|
+
from django.urls.resolvers import RegexPattern, RoutePattern, URLResolver
|
|
5
|
+
|
|
6
|
+
import asyncio, logging, threading
|
|
7
|
+
from asgiref.sync import SyncToAsync
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
def get_django_settings():
|
|
12
|
+
"""
|
|
13
|
+
Get the Django settings
|
|
14
|
+
"""
|
|
15
|
+
try:
|
|
16
|
+
return settings
|
|
17
|
+
except:
|
|
18
|
+
pass
|
|
19
|
+
try:
|
|
20
|
+
from django.conf import settings
|
|
21
|
+
if settings.configured:
|
|
22
|
+
return settings
|
|
23
|
+
except:
|
|
24
|
+
pass
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def get_config(config=None):
|
|
29
|
+
"""
|
|
30
|
+
Get the configuration for the socket server
|
|
31
|
+
"""
|
|
32
|
+
# If the config is passed, return it.
|
|
33
|
+
if config is not None:
|
|
34
|
+
return config
|
|
35
|
+
# If the config is not passed, try to get it from the Django settings
|
|
36
|
+
django_settings = get_django_settings()
|
|
37
|
+
if hasattr(django_settings, "DJANGO_SOCKETS_CONFIG"):
|
|
38
|
+
return django_settings.DJANGO_SOCKETS_CONFIG
|
|
39
|
+
# If nothing has been returned yet, return a default configuration
|
|
40
|
+
return {"hosts": [{"address": "redis://0.0.0.0:6379"}]}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def run_in_thread(command, *args, **kwargs):
|
|
44
|
+
"""
|
|
45
|
+
Takes in a synchronous command along with args and kwargs and runs it in a background
|
|
46
|
+
thread that is not tied to the websocket connection.
|
|
47
|
+
|
|
48
|
+
This will be terminated when the larger daphne server is terminated
|
|
49
|
+
"""
|
|
50
|
+
thread = threading.Thread(
|
|
51
|
+
target=command, args=args, kwargs=kwargs, daemon=True
|
|
52
|
+
)
|
|
53
|
+
thread.start()
|
|
54
|
+
return thread
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def start_event_loop_thread(loop):
|
|
58
|
+
"""
|
|
59
|
+
Starts the event loop in a new thread
|
|
60
|
+
"""
|
|
61
|
+
asyncio.set_event_loop(loop)
|
|
62
|
+
loop.run_forever()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def ensure_loop_running(loop=None):
|
|
66
|
+
"""
|
|
67
|
+
Starts the event loop in a new thread and returns the thread
|
|
68
|
+
"""
|
|
69
|
+
loop = loop if loop is not None else asyncio.get_event_loop()
|
|
70
|
+
if not loop.is_running():
|
|
71
|
+
try:
|
|
72
|
+
thread = run_in_thread(start_event_loop_thread, loop)
|
|
73
|
+
except:
|
|
74
|
+
logger.log(logging.ERROR, "Event Loop already running")
|
|
75
|
+
return loop
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# The following code is copied directly from Django Channels (channels/db.py)
|
|
79
|
+
# Begin Code Copy:
|
|
80
|
+
################################################################################
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class DatabaseSyncToAsync(SyncToAsync):
|
|
84
|
+
"""
|
|
85
|
+
SyncToAsync version that cleans up old database connections when it exits.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def thread_handler(self, loop, *args, **kwargs):
|
|
89
|
+
close_old_connections()
|
|
90
|
+
try:
|
|
91
|
+
return super().thread_handler(loop, *args, **kwargs)
|
|
92
|
+
finally:
|
|
93
|
+
close_old_connections()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# The class is TitleCased, but we want to encourage use as a callable/decorator
|
|
97
|
+
database_sync_to_async = DatabaseSyncToAsync
|
|
98
|
+
################################################################################
|
|
99
|
+
# End Code Copy:
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# The following code is copied directly from Django Channels (channels/routing.py)
|
|
103
|
+
# Begin Code Copy:
|
|
104
|
+
################################################################################
|
|
105
|
+
class ProtocolTypeRouter:
|
|
106
|
+
"""
|
|
107
|
+
Takes a mapping of protocol type names to other Application instances,
|
|
108
|
+
and dispatches to the right one based on protocol name (or raises an error)
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
def __init__(self, application_mapping):
|
|
112
|
+
self.application_mapping = application_mapping
|
|
113
|
+
|
|
114
|
+
async def __call__(self, scope, receive, send):
|
|
115
|
+
if scope["type"] in self.application_mapping:
|
|
116
|
+
application = self.application_mapping[scope["type"]]
|
|
117
|
+
return await application(scope, receive, send)
|
|
118
|
+
else:
|
|
119
|
+
raise ValueError(
|
|
120
|
+
"No application configured for scope type %r" % scope["type"]
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class URLRouter:
|
|
125
|
+
"""
|
|
126
|
+
Routes to different applications/consumers based on the URL path.
|
|
127
|
+
|
|
128
|
+
Works with anything that has a ``path`` key, but intended for WebSocket
|
|
129
|
+
and HTTP. Uses Django's django.urls objects for resolution -
|
|
130
|
+
path() or re_path().
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
#: This router wants to do routing based on scope[path] or
|
|
134
|
+
#: scope[path_remaining]. ``path()`` entries in URLRouter should not be
|
|
135
|
+
#: treated as endpoints (ended with ``$``), but similar to ``include()``.
|
|
136
|
+
_path_routing = True
|
|
137
|
+
|
|
138
|
+
def __init__(self, routes):
|
|
139
|
+
self.routes = routes
|
|
140
|
+
|
|
141
|
+
for route in self.routes:
|
|
142
|
+
# The inner ASGI app wants to do additional routing, route
|
|
143
|
+
# must not be an endpoint
|
|
144
|
+
if getattr(route.callback, "_path_routing", False) is True:
|
|
145
|
+
pattern = route.pattern
|
|
146
|
+
if isinstance(pattern, RegexPattern):
|
|
147
|
+
arg = pattern._regex
|
|
148
|
+
elif isinstance(pattern, RoutePattern):
|
|
149
|
+
arg = pattern._route
|
|
150
|
+
else:
|
|
151
|
+
raise ValueError(
|
|
152
|
+
f"Unsupported pattern type: {type(pattern)}"
|
|
153
|
+
)
|
|
154
|
+
route.pattern = pattern.__class__(
|
|
155
|
+
arg, pattern.name, is_endpoint=False
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
if not route.callback and isinstance(route, URLResolver):
|
|
159
|
+
raise ImproperlyConfigured(
|
|
160
|
+
"%s: include() is not supported in URLRouter. Use nested"
|
|
161
|
+
" URLRouter instances instead." % (route,)
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
async def __call__(self, scope, receive, send):
|
|
165
|
+
# Get the path
|
|
166
|
+
path = scope.get("path_remaining", scope.get("path", None))
|
|
167
|
+
if path is None:
|
|
168
|
+
raise ValueError(
|
|
169
|
+
"No 'path' key in connection scope, cannot route URLs"
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
if "path_remaining" not in scope:
|
|
173
|
+
# We are the outermost URLRouter, so handle root_path if present.
|
|
174
|
+
root_path = scope.get("root_path", "")
|
|
175
|
+
if root_path and not path.startswith(root_path):
|
|
176
|
+
# If root_path is present, path must start with it.
|
|
177
|
+
raise ValueError("No route found for path %r." % path)
|
|
178
|
+
path = path[len(root_path) :]
|
|
179
|
+
|
|
180
|
+
# Remove leading / to match Django's handling
|
|
181
|
+
path = path.lstrip("/")
|
|
182
|
+
# Run through the routes we have until one matches
|
|
183
|
+
for route in self.routes:
|
|
184
|
+
try:
|
|
185
|
+
match = route.pattern.match(path)
|
|
186
|
+
if match:
|
|
187
|
+
new_path, args, kwargs = match
|
|
188
|
+
# Add defaults to kwargs from the URL pattern.
|
|
189
|
+
kwargs.update(route.default_args)
|
|
190
|
+
# Add args or kwargs into the scope
|
|
191
|
+
outer = scope.get("url_route", {})
|
|
192
|
+
application = route.callback
|
|
193
|
+
return await application(
|
|
194
|
+
dict(
|
|
195
|
+
scope,
|
|
196
|
+
path_remaining=new_path,
|
|
197
|
+
url_route={
|
|
198
|
+
"args": outer.get("args", ()) + args,
|
|
199
|
+
"kwargs": {**outer.get("kwargs", {}), **kwargs},
|
|
200
|
+
},
|
|
201
|
+
),
|
|
202
|
+
receive,
|
|
203
|
+
send,
|
|
204
|
+
)
|
|
205
|
+
except Resolver404:
|
|
206
|
+
pass
|
|
207
|
+
else:
|
|
208
|
+
if "path_remaining" in scope:
|
|
209
|
+
raise Resolver404("No route found for path %r." % path)
|
|
210
|
+
# We are the outermost URLRouter
|
|
211
|
+
raise ValueError("No route found for path %r." % path)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
################################################################################
|
|
215
|
+
# End Code Copy:
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Connor Makowski
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: django_sockets
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Simplified Django websocket processes designed to work with cloud caches
|
|
5
|
+
Author-email: Connor Makowski <conmak@mit.edu>
|
|
6
|
+
Project-URL: Homepage, https://github.com/connor-makowski/django_sockets
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/connor-makowski/django_sockets/issues
|
|
8
|
+
Project-URL: Documentation, https://connor-makowski.github.io/django_sockets/django_sockets.html
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: django >=5.0.0
|
|
16
|
+
Requires-Dist: asgiref >=3.0.0
|
|
17
|
+
Requires-Dist: redis >=5.0.0
|
|
18
|
+
Requires-Dist: msgpack >=1.0.0
|
|
19
|
+
|
|
20
|
+
# Django Sockets
|
|
21
|
+
[](https://badge.fury.io/py/django_sockets)
|
|
22
|
+
[](https://opensource.org/licenses/MIT)
|
|
23
|
+
|
|
24
|
+
Simplified Django websocket processes designed to work with cloud caches (valkey|redis on single|distributed|serverless)
|
|
25
|
+
|
|
26
|
+
# Setup
|
|
27
|
+
|
|
28
|
+
### General
|
|
29
|
+
|
|
30
|
+
Make sure you have Python 3.10.x (or higher) installed on your system. You can download it [here](https://www.python.org/downloads/).
|
|
31
|
+
|
|
32
|
+
### Installation
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
pip install django_sockets
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
### Other Requirements
|
|
39
|
+
|
|
40
|
+
- Make sure a cache (valkey or redis) is setup and accessible from your server.
|
|
41
|
+
|
|
42
|
+
- To run one locally (using docker) for testing, you can use the following command:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
docker run -d -p 6379:6379 --name django_sockets_cache valkey/valkey:7
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
- To run the container after it has been created, you can use the following command:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
docker start django_sockets_cache
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
- To kill the container later, you can use the following command:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
docker kill django_sockets_cache
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
# Usage
|
|
61
|
+
|
|
62
|
+
Low level docs: https://connor-makowski.github.io/django_sockets/django_sockets.html
|
|
63
|
+
|
|
64
|
+
## Implement in a Django Project
|
|
65
|
+
|
|
66
|
+
### TODO
|
|
67
|
+
|
|
68
|
+
## Examples Without Django
|
|
69
|
+
|
|
70
|
+
### Example Subscribing & Broadcasting
|
|
71
|
+
```py
|
|
72
|
+
from django_sockets.sockets import BaseSocketServer
|
|
73
|
+
import asyncio, time
|
|
74
|
+
|
|
75
|
+
DJANGO_SOCKETS_CONFIG = {
|
|
76
|
+
"hosts": [
|
|
77
|
+
{"address": f"redis://0.0.0.0:6379"}
|
|
78
|
+
],
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
# Override the send method to print the data being sent
|
|
82
|
+
# instead of sending it over a non existent websocket connection
|
|
83
|
+
async def send(ws_data):
|
|
84
|
+
print("WS SENDING:", ws_data)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# Create a receive queue to simulate receiving messages from a websocket client
|
|
88
|
+
base_receive = asyncio.Queue()
|
|
89
|
+
# Create a base socket server with a scope of {}
|
|
90
|
+
base_socket_server = BaseSocketServer(scope={}, receive=base_receive.get, send=send, config=DJANGO_SOCKETS_CONFIG)
|
|
91
|
+
# Start the listeners for the base socket server
|
|
92
|
+
base_socket_server.start_listeners()
|
|
93
|
+
# Subscribe to the test_channel
|
|
94
|
+
base_socket_server.subscribe("test_channel")
|
|
95
|
+
# Broadcast a message to the test_channel
|
|
96
|
+
base_socket_server.broadcast("test_channel", "test message")
|
|
97
|
+
# Give the async functions a small amount of time to complete
|
|
98
|
+
time.sleep(.5)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
#=> Output:
|
|
102
|
+
#=> WS SENDING: {'type': 'websocket.send', 'text': '"test message"'}
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
### Example Handle Websocket Messages
|
|
106
|
+
```py
|
|
107
|
+
from django_sockets.sockets import BaseSocketServer
|
|
108
|
+
import asyncio, time
|
|
109
|
+
|
|
110
|
+
DJANGO_SOCKETS_CONFIG = {
|
|
111
|
+
"hosts": [
|
|
112
|
+
{"address": f"redis://0.0.0.0:6379"}
|
|
113
|
+
],
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
class CustomSocketServer(BaseSocketServer):
|
|
117
|
+
def receive(self, data):
|
|
118
|
+
"""
|
|
119
|
+
When a data message is received from a websocket client:
|
|
120
|
+
- Print the data
|
|
121
|
+
- Broadcast the data to a channel (the same channel that the socket server is subscribed to)
|
|
122
|
+
|
|
123
|
+
Normally you would want to override the receive method to do any server side processing of the data that is received
|
|
124
|
+
then broadcast any changes back to relevant channels.
|
|
125
|
+
"""
|
|
126
|
+
print("WS RECEIVED: ", data)
|
|
127
|
+
print(f"BROADCASTING TO '{self.scope['username']}'")
|
|
128
|
+
self.broadcast(self.scope['username'], data)
|
|
129
|
+
|
|
130
|
+
def connect(self):
|
|
131
|
+
"""
|
|
132
|
+
When the websocket connects, subscribe to the channel of the user.
|
|
133
|
+
|
|
134
|
+
This is an important method to override if you want to subscribe to a channel when a user frist connects.
|
|
135
|
+
|
|
136
|
+
Otherwise, you can always subscribe to a channel based on the data that is received in the receive method.
|
|
137
|
+
"""
|
|
138
|
+
print(f"CONNECTED")
|
|
139
|
+
print(f"SUSCRIBING TO '{self.scope['username']}'")
|
|
140
|
+
self.subscribe(self.scope['username'])
|
|
141
|
+
|
|
142
|
+
# Override the send method to print the data being sent
|
|
143
|
+
async def send(data):
|
|
144
|
+
"""
|
|
145
|
+
Normally you would not override the send method, but since we are not actually sending data over a websocket connection
|
|
146
|
+
we are just going to print the data that would be sent.
|
|
147
|
+
|
|
148
|
+
This is useful for testing the socket server without having to actually send data over a websocket connection
|
|
149
|
+
|
|
150
|
+
Note: This only sends the first 64 characters of the data
|
|
151
|
+
"""
|
|
152
|
+
print("WS SENDING:", str(data)[:64])
|
|
153
|
+
|
|
154
|
+
# Create a receive queue to simulate receiving messages from a websocket client
|
|
155
|
+
custom_receive = asyncio.Queue()
|
|
156
|
+
# Create a custom socket server defined above with a scope of {'username':'adam'}, the custom_receive queue, and the send method defined above
|
|
157
|
+
custom_socket_server = CustomSocketServer(scope={'username':'adam'}, receive=custom_receive.get, send=send)
|
|
158
|
+
# Start the listeners for the custom socket server
|
|
159
|
+
# - Websocket Listener - Listens for websocket messages
|
|
160
|
+
# - Broadcast Listener - Listens for messages that were broadcasted to a channel that the socket server is subscribed to
|
|
161
|
+
custom_socket_server.start_listeners()
|
|
162
|
+
# Give the async functions a small amount of time to complete
|
|
163
|
+
time.sleep(.1)
|
|
164
|
+
# Simulate a WS connection request
|
|
165
|
+
custom_receive.put_nowait({'type': 'websocket.connect'})
|
|
166
|
+
# Give the async functions a small amount of time to complete
|
|
167
|
+
time.sleep(.1)
|
|
168
|
+
# Simulate a message being received from a WS client
|
|
169
|
+
# This will call the receive method which is defined above
|
|
170
|
+
custom_receive.put_nowait({'type': 'websocket.receive', 'text': '{"data": "test"}'})
|
|
171
|
+
# Give the async functions a small amount of time to complete
|
|
172
|
+
time.sleep(.1)
|
|
173
|
+
# Simulate a WS disconnect request
|
|
174
|
+
custom_receive.put_nowait({'type': 'websocket.disconnect'})
|
|
175
|
+
# Give the async functions a small amount of time to complete
|
|
176
|
+
time.sleep(.1)
|
|
177
|
+
# Simulate a message being received from a WS client after the connection has been closed
|
|
178
|
+
# This will not do anything since the connection has been closed and the listeners have been killed
|
|
179
|
+
custom_receive.put_nowait({'type': 'websocket.receive', 'text': '{"data_after_close": "test"}'})
|
|
180
|
+
# Give the async functions a small amount of time to complete
|
|
181
|
+
time.sleep(.1)
|
|
182
|
+
|
|
183
|
+
#=> Output:
|
|
184
|
+
#=> WS SENDING: {'type': 'websocket.accept'}
|
|
185
|
+
#=> CONNECTED
|
|
186
|
+
#=> SUSCRIBING TO 'adam'
|
|
187
|
+
#=> WS RECEIVED: {'data': 'test'}
|
|
188
|
+
#=> BROADCASTING TO 'adam'
|
|
189
|
+
#=> WS SENDING: {'type': 'websocket.send', 'text': '{"data": "test"}'}
|
|
190
|
+
|
|
191
|
+
```
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
django_sockets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
django_sockets/broadcaster.py,sha256=rX7bGA1-TS1mCt2R0Nv66Ts7XKXvalV8kl2ula8fWQ4,3182
|
|
3
|
+
django_sockets/middleware.py,sha256=2tOP0XoIEZ3towGxTzCK_5HW1oej4rmFdf5DJF99xLY,1309
|
|
4
|
+
django_sockets/pubsub.py,sha256=mf_v7w8efnRxYQhFn7Df6w1jJllLZ7S_m30fRzIJzJg,12548
|
|
5
|
+
django_sockets/sockets.py,sha256=D9GLivaSybjyPbsp9y0Mi7AWGbLsUMvKOtT4jK825vk,8367
|
|
6
|
+
django_sockets/utils.py,sha256=mySD-fOwDuCUGmSZ19yhe6eGp9QhNbZCvDceBk9h7DY,7503
|
|
7
|
+
django_sockets-1.0.0.dist-info/LICENSE,sha256=lWsmWo1t9v94zSBfnvvyFFVyAKworr9FQWvXDmk-Peo,1072
|
|
8
|
+
django_sockets-1.0.0.dist-info/METADATA,sha256=k8E06XeHvS95NwqkdfjY9_6mdhsvKSw2N3HwyODMlss,6995
|
|
9
|
+
django_sockets-1.0.0.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
10
|
+
django_sockets-1.0.0.dist-info/top_level.txt,sha256=V4H73fZaPpKLIl3GqUGNQ-QW0lqv7B8NamEOJPh-ZCg,15
|
|
11
|
+
django_sockets-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
django_sockets
|