dsmq 0.2.0__py3-none-any.whl

dsmq/__init__.py

@@ -0,0 +1,2 @@
+def hello() -> str:
+    return "Hello from dsmq!"

dsmq/demo_linux.py

@@ -0,0 +1,25 @@
+# Heads up: This script only works on Linux because of use of "fork" method.
+import multiprocessing as mp
+import dsmq
+import example_get_client
+import example_put_client
+
+mp.set_start_method("fork")
+
+HOST = "127.0.0.1"
+PORT = 25252
+
+
+def test_server_with_clients():
+    p_server = mp.Process(target=dsmq.run, args=(HOST, PORT))
+    p_server.start()
+
+    p_putter = mp.Process(target=example_put_client.run, args=(HOST, PORT, 20))
+    p_getter = mp.Process(target=example_get_client.run, args=(HOST, PORT, 20))
+
+    p_putter.start()
+    p_getter.start()
+
+
+if __name__ == "__main__":
+    test_server_with_clients()

dsmq/dsmq.py

@@ -0,0 +1,160 @@
+import json
+import socket
+import sqlite3
+import sys
+from threading import Thread
+import time
+
+N_RETRIES = 5
+FIRST_RETRY = 0.01  # seconds
+
+
+def run(host="127.0.0.1", port=30008):
+    sqlite_conn = sqlite3.connect("file:mem1?mode=memory&cache=shared")
+    cursor = sqlite_conn.cursor()
+
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS messages (timestamp DOUBLE, topic TEXT, message TEXT)
+    """)
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        # Setting this socket option to re-use the address,
+        # even if it's already in use.
+        # This is helpful in recovering from crashes where things didn't
+        # shut down properly.
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+        s.bind((host, port))
+        s.listen()
+
+        print()
+        print(f"Server started at {host} on port {port}.")
+        print("Waiting for clients...")
+
+        while True:
+            socket_conn, addr = s.accept()
+            print(f"Connected by {addr}")
+            Thread(target=handle_socket, args=(socket_conn,)).start()
+
+    sqlite_conn.close()
+
+
+def handle_socket(socket_conn):
+    sqlite_conn = sqlite3.connect("file:mem1?mode=memory&cache=shared")
+    cursor = sqlite_conn.cursor()
+
+    creation_time = time.time()
+    last_read = {}
+
+    while True:
+        data = socket_conn.recv(1024)
+        # Check whether the connection has been terminated
+        if not data:
+            break
+
+        msg_str = data.decode("utf-8")
+        try:
+            # print("dsmq received ", msg_str)
+            msg = json.loads(msg_str)
+        except json.decoder.JSONDecodeError:
+            print("Message must be json-friendly")
+            print(f"    Received: {msg}")
+            continue
+
+        topic = msg["topic"]
+        timestamp = time.time()
+
+        if msg["action"] == "put":
+            msg["timestamp"] = timestamp
+
+            # This block allows for multiple retries if the database
+            # is busy.
+            for i_retry in range(N_RETRIES):
+                try:
+                    cursor.execute(
+                        """
+INSERT INTO messages (timestamp, topic, message)
+VALUES (:timestamp, :topic, :message)
+                        """,
+                        (msg),
+                    )
+                    sqlite_conn.commit()
+                except sqlite3.OperationalError:
+                    wait_time = FIRST_RETRY * 2**i_retry
+                    time.sleep(wait_time)
+                    continue
+                break
+
+        elif msg["action"] == "get":
+            try:
+                last_read_time = last_read[topic]
+            except KeyError:
+                last_read[topic] = creation_time
+                last_read_time = last_read[topic]
+            msg["last_read_time"] = last_read_time
+
+            # This block allows for multiple retries if the database
+            # is busy.
+            for i_retry in range(N_RETRIES):
+                try:
+                    cursor.execute(
+                        """
+SELECT message,
+timestamp
+FROM messages,
+(
+SELECT MIN(timestamp) AS min_time
+FROM messages
+WHERE topic = :topic
+    AND timestamp > :last_read_time
+) a
+WHERE topic = :topic
+AND timestamp = a.min_time
+                        """,
+                        msg,
+                    )
+                except sqlite3.OperationalError:
+                    wait_time = FIRST_RETRY * 2**i_retry
+                    time.sleep(wait_time)
+                    continue
+                break
+
+            try:
+                result = cursor.fetchall()[0]
+                message = result[0]
+                timestamp = result[1]
+                last_read[topic] = timestamp
+            except IndexError:
+                # Handle the case where no results are returned
+                message = ""
+
+            msg = json.dumps({"message": message})
+            socket_conn.sendall(bytes(msg, "utf-8"))
+        else:
+            print("Action must either be 'put' or 'get'")
+
+    sqlite_conn.close()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 3:
+        host = sys.argv[1]
+        port = int(sys.argv[2])
+        run(host=host, port=port)
+    elif len(sys.argv) == 2:
+        host = sys.argv[1]
+        run(host=host)
+    elif len(sys.argv) == 1:
+        run()
+    else:
+        print(
+            """
+Try one of these:
+$ python3 dsmq.py
+
+$ python3 dsmq.py 127.0.0.1
+
+$ python3 dsmq.py 127.0.0.1 25853
+
+            """
+        )

dsmq/example_get_client.py

@@ -0,0 +1,22 @@
+import json
+import socket
+import time
+
+
+def run(host="127.0.0.1", port=30008, n_iter=1000):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.connect((host, port))
+
+        for i in range(n_iter):
+            time.sleep(1)
+            msg = json.dumps({"action": "get", "topic": "greetings"})
+            s.sendall(bytes(msg, "utf-8"))
+            data = s.recv(1024)
+            if not data:
+                raise RuntimeError("Connection terminated by server")
+            msg_str = data.decode("utf-8")
+            print(f"client received {json.loads(msg_str)}")
+
+
+if __name__ == "__main__":
+    run()

dsmq/example_put_client.py

@@ -0,0 +1,19 @@
+import json
+import socket
+import time
+
+
+def run(host="127.0.0.1", port=30008, n_iter=1000):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.connect((host, port))
+
+        for i in range(n_iter):
+            time.sleep(1)
+            note = f"{i}. Hello, world"
+            msg = json.dumps({"action": "put", "topic": "greetings", "message": note})
+            s.sendall(bytes(msg, "utf-8"))
+            print(f"client sent {msg}")
+
+
+if __name__ == "__main__":
+    run()

dsmq/example_server.py

@@ -0,0 +1,3 @@
+from dsmq import dsmq
+
+dsmq.run(host="127.0.0.1", port=12345)

dsmq-0.2.0.dist-info/METADATA

@@ -0,0 +1,156 @@
+Metadata-Version: 2.3
+Name: dsmq
+Version: 0.2.0
+Summary: A dead simple message queue
+License-File: LICENSE
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# Dead Simple Message Queue
+
+## What it does
+
+Part mail room, part bulletin board, dsmq is a central location for sharing messages
+between processes, even when they are running on computers scattered around the world.
+
+Its defining characteristic is its bare-bones simplicity.
+
+## How to use it
+
+### Install
+
+```bash
+pip install dsmq
+```
+### Create a dsmq server
+
+As in `src/dsmq/example_server.py`
+
+```python
+from dsmq import dsmq
+
+dsmq.run(host="127.0.0.1", port=12345)
+```
+
+### Add a message to a queue
+
+As in `src/dsmq/example_put_client.py`
+
+```python
+import json
+import socket
+
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    s.connect(("127.0.0.1", 12345))
+    message_content = {"action": "put", "topic": "greetings", "message": "Hello!"}
+    msg = json.dumps(message_content)
+    s.sendall(bytes(msg, "utf-8"))
+```
+
+### Read a message from a queue
+
+As in `src/dsmq/example_get_client.py`
+
+```python
+import json
+import socket
+
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    s.connect(("127.0.0.1", 12345))
+
+    for i in range(n_iter):
+        request_message_content = {"action": "get", "topic": "greetings"}
+        request_msg = json.dumps(request_message_content)
+        s.sendall(bytes(reequest_msg, "utf-8"))
+
+        reply_msg = s.recv(1024)
+        if not reply_msg:
+            raise RuntimeError("Connection terminated by server")
+        reply_msg_content = reply_msg.decode("utf-8")
+```
+
+### Demo
+
+1. Open 3 separate terminal windows.
+1. In the first, run `src/dsmq/dsmq.py`.
+1. In the second, run `src/dsmq/example_put_client.py`.
+1. In the third, run `src/dsmq/example_get_client.py`.
+
+
+## How it works
+
+### Expected behavior and limitations
+
+- Many clients can read messages of the same topic. It is a one-to-many
+pulication model.
+
+- A client will not be able to read any of the messages that were put into
+a queue before it connected.
+
+- A client will get the oldest message available on a requested topic.
+Queues are first-in-first-out.
+
+- Put and get operations are fairly quick--less than 100 $`\mu`$s of processing
+time plus any network latency--so it can comfortably handle operations at
+hundreds of Hz. But if you try to have several clients reading and writing
+at 1 kHz or more, you may overload the queue.
+
+- The queue is backed by an in-memory SQLite database. If your message volumes
+get larger than your RAM, you may reach an out-of-memory condition.
+
+
+# API Reference and Cookbook
+[[source](https://github.com/brohrer/dsmq/blob/main/src/dsmq/dsmq.py)]
+
+### Start a server
+
+```python
+run(host="127.0.0.1", port=30008)
+```
+
+Kicks off the mesage queue server. This process will be the central exchange
+for all incoming and outgoing messages.
+- `host` (str), IP address on which the server will be visible and
+- `port` (int), port. These will be used by all clients.
+Non-privileged ports are numbered 1024 and higher.
+
+### Open a connection from a client
+
+```python
+import socket
+
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    s.connect((host, port ))
+```
+
+### Add a message to a queue
+
+```python
+import json
+msg = json.dumps({
+    "action": "put",
+    "topic": <queue-name>,
+    "message": <message-content>
+})
+s.sendall(bytes(msg, "utf-8"))
+```
+
+- `s`, the socket connection to the server
+- `<queue-name>` (str), a name for the queue where the message will be added
+- `<message-content>` (str), whatever message content you want
+
+Place `message-content` into the queue named `queue-name`.
+If the queue doesn't exist yet, create it.
+
+### Get a message from a queue
+
+```python
+request_msg = json.dumps({"action": "get", "topic": <queue-name>})
+s.sendall(bytes(request_msg, "utf-8"))
+data = s.recv(1024)
+msg_str = data.decode("utf-8")
+```
+
+Get the oldest eligible message from the queue named `<queue-name>`.
+The client is only elgibile to receive messages added after it
+connected to the server.

dsmq-0.2.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+dsmq/__init__.py,sha256=YCgbnQAk8YbtHRyMcU0v2O7RdRhPhlT-vS_q40a7Q6g,50
+dsmq/demo_linux.py,sha256=D5NjEvPG-fTGRRbYzMJXgtwfrigOt_eaR6D0AozBNbA,613
+dsmq/dsmq.py,sha256=EJW6tMeG57QYBneqBEM4R_5CrtG-Lx2qz0D1BM0YtjI,4452
+dsmq/example_get_client.py,sha256=ZWZrAu7sIEn8VwCK09cTwsm1XnF3Jkor3r7Gl4ljUMw,630
+dsmq/example_put_client.py,sha256=bSjSJZsFZB6inVGUlTnJn0jgrpVNXvrN46STTvgwCzs,497
+dsmq/example_server.py,sha256=UM69GN7uORDetXQM8LlW68nrA-1i7Jgp2dZREVEqxaU,62
+dsmq/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dsmq-0.2.0.dist-info/METADATA,sha256=TW6tM_oF2LuyRrmOBSBTHFCkJvNDA8O3QfuEtgD1-S8,4055
+dsmq-0.2.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+dsmq-0.2.0.dist-info/licenses/LICENSE,sha256=3Yu1mAp5VsKmnDtzkiOY7BdmrLeNwwZ3t6iWaLnlL0Y,1071
+dsmq-0.2.0.dist-info/RECORD,,

dsmq-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.25.0
+Root-Is-Purelib: true
+Tag: py3-none-any

dsmq-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Brandon Rohrer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.