dsmq 1.3.1__py3-none-any.whl → 1.4.1__py3-none-any.whl

dsmq/client.py

@@ -18,6 +18,7 @@ def connect(host=_default_host, port=_default_port, verbose=False):
 class DSMQClientSideConnection:
     def __init__(self, host, port, verbose=False):
         self.uri = f"ws://{host}:{port}"
+        self.port = port
         self.verbose = verbose
         if self.verbose:
             print(f"Connecting to dsmq server at {self.uri}")
@@ -40,7 +41,11 @@ class DSMQClientSideConnection:
 
     def get(self, topic):
         msg = {"action": "get", "topic": topic}
-        self.websocket.send(json.dumps(msg))
+        try:
+            self.websocket.send(json.dumps(msg))
+        except ConnectionClosedError:
+            return ""
+
         try:
             msg_text = self.websocket.recv()
         except ConnectionClosedError:
@@ -50,6 +55,29 @@ class DSMQClientSideConnection:
         msg = json.loads(msg_text)
         return msg["message"]
 
+    def get_latest(self, topic):
+        """
+        A variant of `get()` that grabs the latest available message
+        (if there is one) rather than grabbing the oldest unread message.
+        It will not go back to read older ones on subsequent calls;
+        it will leave them unread.
+        """
+        msg = {"action": "get_latest", "topic": topic}
+        try:
+            self.websocket.send(json.dumps(msg))
+        except ConnectionClosedError:
+            return ""
+
+        try:
+            msg_text = self.websocket.recv()
+        except ConnectionClosedError:
+            self.close()
+            return ""
+
+        msg = json.loads(msg_text)
+
+        return msg["message"]
+
     def get_wait(self, topic):
         """
         A variant of `get()` that retries a few times until it gets
@@ -65,7 +93,10 @@ class DSMQClientSideConnection:
 
     def put(self, topic, msg_body):
         msg_dict = {"action": "put", "topic": topic, "message": msg_body}
-        self.websocket.send(json.dumps(msg_dict))
+        try:
+            self.websocket.send(json.dumps(msg_dict))
+        except ConnectionClosedError:
+            return
 
     def shutdown_server(self):
         msg_dict = {"action": "shutdown", "topic": ""}

dsmq/demo.py

@@ -1,7 +1,16 @@
 import multiprocessing as mp
-from dsmq.server import serve
-import dsmq.example_get_client
-import dsmq.example_put_client
+
+# spawn is the default method on macOS,
+# starting in Python 3.14 it will be the default in Linux too.
+try:
+    mp.set_start_method("spawn")
+except RuntimeError:
+    # Will throw an error if the start method has alraedy been set.
+    pass
+
+from dsmq.server import serve  # noqa: E402
+import dsmq.example_get_client  # noqa: E402
+import dsmq.example_put_client  # noqa: E402
 
 HOST = "127.0.0.1"
 PORT = 25252

dsmq/server.py

@@ -9,10 +9,12 @@ from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK
 
 _default_host = "127.0.0.1"
 _default_port = 30008
-_n_retries = 20
-_first_retry = 0.005  # seconds
-_time_between_cleanup = 0.05  # seconds
 _max_queue_length = 10
+_shutdown_pause = 1.0  # seconds
+_time_between_cleanup = 3.0  # seconds
+_time_to_keep = 0.3  # seconds
+# _time_between_cleanup = 60.0  # seconds
+# _time_to_keep = 10.0  # seconds
 
 # Make this global so it's easy to share
 dsmq_server = None
@@ -42,10 +44,15 @@ def serve(
     # and keep long-term latency more predictable.
     # These also make it more susceptible to corruption during shutdown,
     # but since dsmq is meant to be ephemeral, that's not a concern.
-    cursor.execute("PRAGMA journal_mode = OFF")
-    cursor.execute("PRAGMA synchronous = OFF")
-    cursor.execute("PRAGMA secure_delete = OFF")
-    cursor.execute("PRAGMA temp_store = MEMORY")
+    # See https://www.sqlite.org/pragma.html
+    # After playing around with them, I'm not sure these have any noticeable effect.
+    #
+    # cursor.execute("PRAGMA journal_mode = OFF")
+    # cursor.execute("PRAGMA journal_mode = MEMORY")
+    # cursor.execute("PRAGMA synchronous = NORMAL")
+    # cursor.execute("PRAGMA synchronous = OFF")
+    # cursor.execute("PRAGMA secure_delete = OFF")
+    # cursor.execute("PRAGMA temp_store = MEMORY")
 
     cursor.execute("""
 CREATE TABLE IF NOT EXISTS messages (timestamp DOUBLE, topic TEXT, message TEXT)
@@ -56,36 +63,33 @@ CREATE TABLE IF NOT EXISTS messages (timestamp DOUBLE, topic TEXT, message TEXT)
     # and a method of last resort.
     global dsmq_server
 
-    for i_retry in range(_n_retries):
-        try:
+    try:
+        with ws_serve(request_handler, host, port) as dsmq_server:
+            dsmq_server.serve_forever()
+
+    except OSError:
+        # Catch the case where the address is already in use
+        if verbose:
+            print()
+            print(f"Found a dsmq server already running on {host} on port {port}.")
+            print("    Closing it down.")
+
+            def shutdown_gracefully(server_to_shutdown):
+                server_to_shutdown.shutdown()
+
+            Thread(target=shutdown_gracefully, args=(dsmq_server,)).start()
+            time.sleep(_shutdown_pause)
+
             with ws_serve(request_handler, host, port) as dsmq_server:
                 dsmq_server.serve_forever()
 
-            if verbose:
-                print()
-                print(f"Server started at {host} on port {port}.")
-                print("Waiting for clients...")
-
-            break
-
-        except OSError:
-            # Catch the case where the address is already in use
-            if verbose:
-                print()
-                if i_retry < _n_retries - 1:
-                    print(f"Couldn't start dsmq server on {host} on port {port}.")
-                    print(f"    Trying again ({i_retry}) ...")
-                else:
-                    print()
-                    print(f"Failed to start dsmq server on {host} on port {port}.")
-                    print()
-                    raise
-
-            wait_time = _first_retry * 2**i_retry
-            time.sleep(wait_time)
+    if verbose:
+        print()
+        print(f"Server started at {host} on port {port}.")
+        print("Waiting for clients...")
 
     sqlite_conn.close()
-    time.sleep(_time_between_cleanup)
+    time.sleep(_shutdown_pause)
     cleanup_temp_files()
 
 
@@ -121,9 +125,10 @@ def request_handler(websocket):
         for msg_text in websocket:
             msg = json.loads(msg_text)
             topic = msg["topic"]
+            action = msg["action"]
             timestamp = time.time()
 
-            if msg["action"] == "put":
+            if action == "put":
                 msg["timestamp"] = timestamp
 
                 try:
@@ -138,7 +143,7 @@ def request_handler(websocket):
                 except sqlite3.OperationalError:
                     pass
 
-            elif msg["action"] == "get":
+            elif action == "get":
                 try:
                     last_read_time = last_read_times[topic]
                 except KeyError:
@@ -151,15 +156,46 @@ def request_handler(websocket):
                         """
                         SELECT message,
                         timestamp
-                        FROM messages,
-                        (
-                        SELECT MIN(timestamp) AS min_time
                         FROM messages
                         WHERE topic = :topic
                         AND timestamp > :last_read_time
-                        ) a
+                        ORDER BY timestamp ASC
+                        LIMIT 1
+                        """,
+                        msg,
+                    )
+                except sqlite3.OperationalError:
+                    pass
+
+                try:
+                    result = cursor.fetchall()[0]
+                    message = result[0]
+                    timestamp = result[1]
+                    last_read_times[topic] = timestamp
+                except IndexError:
+                    # Handle the case where no results are returned
+                    message = ""
+
+                websocket.send(json.dumps({"message": message}))
+
+            elif action == "get_latest":
+                try:
+                    last_read_time = last_read_times[topic]
+                except KeyError:
+                    last_read_times[topic] = client_creation_time
+                    last_read_time = last_read_times[topic]
+                msg["last_read_time"] = last_read_time
+
+                try:
+                    cursor.execute(
+                        """
+                        SELECT message,
+                        timestamp
+                        FROM messages
                         WHERE topic = :topic
-                        AND timestamp = a.min_time
+                        AND timestamp > :last_read_time
+                        ORDER BY timestamp DESC
+                        LIMIT 1;
                         """,
                         msg,
                     )
@@ -177,25 +213,41 @@ def request_handler(websocket):
 
                 websocket.send(json.dumps({"message": message}))
 
-            elif msg["action"] == "shutdown":
+            elif action == "shutdown":
                 # Run this from a separate thread to prevent deadlock
                 global dsmq_server
 
                 def shutdown_gracefully(server_to_shutdown):
                     server_to_shutdown.shutdown()
-                    # cleanup_temp_files()
 
                 Thread(target=shutdown_gracefully, args=(dsmq_server,)).start()
                 break
             else:
                 raise RuntimeWarning(
-                    "dsmq client action must either be 'put', 'get', or 'shutdown'"
+                    "dsmq client action must either be\n"
+                    + "'put', 'get', 'get_wait', 'get_latest', or 'shutdown'"
                 )
 
             # Periodically clean out messages to keep individual queues at
             # a manageable length and the overall mq small.
             if time.time() - time_of_last_purge > _time_between_cleanup:
+                cutoff_time = time.time() - _time_to_keep
                 try:
+                    cursor.execute(
+                        """
+                        DELETE
+                        FROM messages
+                        WHERE topic = :topic
+                        AND timestamp < :cutoff_time
+                        """,
+                        {
+                            "cutoff_time": cutoff_time,
+                            "topic": topic,
+                        },
+                    )
+                    sqlite_conn.commit()
+                    time_of_last_purge = time.time()
+
                     cursor.execute(
                         """
                         DELETE
@@ -219,6 +271,7 @@ def request_handler(websocket):
                     )
                     sqlite_conn.commit()
                     time_of_last_purge = time.time()
+
                 except sqlite3.OperationalError:
                     # Database may be locked. Try again next time.
                     pass

dsmq/tests/integration_test.py

@@ -1,10 +1,17 @@
 import multiprocessing as mp
+
+try:
+    # spawn is the default method on macOS,
+    # starting in Python 3.14 it will be the default in Linux too.
+    mp.set_start_method("spawn")
+except RuntimeError:
+    # Will throw an error if the start method has alraedy been set.
+    pass
+
 import time
 from dsmq.server import serve
 from dsmq.client import connect
 
-# spawn is the default method on macOS
-# mp.set_start_method('spawn')
 
 host = "127.0.0.1"
 port = 30303
@@ -53,6 +60,10 @@ def test_write_one_read_one():
     write_client = connect(host, port)
     read_client = connect(host, port)
 
+    msg = read_client.get("test")
+
+    assert msg == ""
+
     write_client.put("test", "test_msg")
 
     # It takes a moment for the write to complete
@@ -61,6 +72,10 @@ def test_write_one_read_one():
 
     assert msg == "test_msg"
 
+    msg = read_client.get("test")
+
+    assert msg == ""
+
     write_client.shutdown_server()
     write_client.close()
     read_client.close()
@@ -83,6 +98,29 @@ def test_get_wait():
     read_client.close()
 
 
+def test_get_latest():
+    p_server = mp.Process(target=serve, args=(host, port))
+    p_server.start()
+    write_client = connect(host, port)
+    read_client = connect(host, port)
+
+    for i in range(5):
+        write_client.put("test", f"test_msg {i}")
+
+    time.sleep(_pause)
+    msg = read_client.get_latest("test")
+
+    assert msg == "test_msg 4"
+
+    msg = read_client.get_latest("test")
+
+    assert msg == ""
+
+    write_client.shutdown_server()
+    write_client.close()
+    read_client.close()
+
+
 def test_multitopics():
     p_server = mp.Process(target=serve, args=(host, port))
     p_server.start()
@@ -183,15 +221,8 @@ def test_speed_writing():
     p_speed_write.start()
     time.sleep(_pause)
 
-    # time_a = time.time()
     write_client.put("test", "test_msg")
-    # time_b = time.time()
     msg = read_client.get_wait("test")
-    # time_c = time.time()
-
-    # write_time = int((time_b - time_a) * 1e6)
-    # read_time = int((time_c - time_b) * 1e6)
-    # print(f"write time: {write_time} us,  read time: {read_time} us")
 
     assert msg == "test_msg"
 
@@ -221,15 +252,8 @@ def test_speed_reading():
     p_speed_read = mp.Process(target=speed_read, args=(stop_flag,))
     p_speed_read.start()
 
-    # time_a = time.time()
     write_client.put("test", "test_msg")
-    # time_b = time.time()
     msg = read_client.get_wait("test")
-    # time_c = time.time()
-
-    # write_time = int((time_b - time_a) * 1e6)
-    # read_time = int((time_c - time_b) * 1e6)
-    # print(f"write time: {write_time} us,  read time: {read_time} us")
 
     assert msg == "test_msg"
 
@@ -238,5 +262,8 @@ def test_speed_reading():
     p_speed_read.kill()
     read_client.close()
     write_client.shutdown_server()
-    # time.sleep(_pause)
     write_client.close()
+
+
+if __name__ == "__main__":
+    test_get_latest()

dsmq/tests/performance_suite.py

@@ -1,4 +1,12 @@
 import multiprocessing as mp
+
+# spawn is the default method on macOS,
+# starting in Python 3.14 it will be the default in Linux too.
+try:
+    mp.set_start_method("spawn")
+except RuntimeError:
+    pass
+
 import time
 
 from dsmq.server import serve
@@ -11,8 +19,8 @@ verbose = False
 _pause = 0.01
 _very_long_pause = 1.0
 
-_n_iter = int(1e3)
-_n_long_char = int(1e4)
+_n_iter = int(1e4)
+_n_long_char = int(1e5)
 
 _short_msg = "q"
 _long_msg = str(["q"] * _n_long_char)

{dsmq-1.3.1.dist-info → dsmq-1.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsmq
-Version: 1.3.1
+Version: 1.4.1
 Summary: A dead simple message queue
 License-File: LICENSE
 Requires-Python: >=3.10
@@ -102,7 +102,8 @@ a queue before it connected.
 - A client will get the oldest message available on a requested topic.
 Queues are first-in-first-out.
 
-- Messages older than 600 seconds will be deleted from the queue.
+- Messages older than a certain age (typically 600 seconds)
+will be deleted from the queue.
 
 - Put and get operations are fairly quick--less than 100 $`\mu`$s of processing
 time plus any network latency--so it can comfortably handle requests at rates of
@@ -152,6 +153,16 @@ connected to the server.
 in the topic, or the topic doesn't yet exist,
 returns `""`.
 
+### `get_latest(topic)`
+
+Get the *most recent* eligible message from the queue named `topic`.
+All the messages older than that in the queue become ineligible and never
+get seen by the client.
+- `topic` (str)
+- returns str, the content of the message. If there was no eligble message
+in the topic, or the topic doesn't yet exist,
+returns `""`.
+
 ### `get_wait(topic)`
 
 A variant of `get()` that retries a few times until it gets

dsmq-1.4.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+dsmq/__init__.py,sha256=YCgbnQAk8YbtHRyMcU0v2O7RdRhPhlT-vS_q40a7Q6g,50
+dsmq/client.py,sha256=a4_6jcF7RAC58And0OlR3vL42FgZ24q2BECS-VF4Ri4,3438
+dsmq/demo.py,sha256=HCnoucawZ2CU0rGipLgsZjVUpP7232mUcugmhf_DjBQ,825
+dsmq/example_get_client.py,sha256=PvAsDGEAH1kVBifLVg2rx8ZxnAZmvzVCvZq13VgpLds,301
+dsmq/example_put_client.py,sha256=QxDc3i7KAjjhpwxRRpI0Ke5KTNSPuBf9kkcGyTvUEaw,353
+dsmq/server.py,sha256=PbmbW95VunRrQM24AdzX9HbqWPixt7XrAINV1yiHpxo,10210
+dsmq/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dsmq/tests/integration_test.py,sha256=zF6v_3UHt5KqveX22gEpCSLcoeOyEWLSy_gMLoytg2U,6303
+dsmq/tests/performance_suite.py,sha256=g95jm0u2sa-1hm6m10aoRgN7_tynXFxU-bimy3V2wZM,5391
+dsmq-1.4.1.dist-info/METADATA,sha256=LuoL1eb8_zH2bDCY7QRlKrZ3ndK0UJ1gXJKZ-4miWYc,5229
+dsmq-1.4.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dsmq-1.4.1.dist-info/licenses/LICENSE,sha256=3Yu1mAp5VsKmnDtzkiOY7BdmrLeNwwZ3t6iWaLnlL0Y,1071
+dsmq-1.4.1.dist-info/RECORD,,

dsmq-1.3.1.dist-info/RECORD

@@ -1,13 +0,0 @@
-dsmq/__init__.py,sha256=YCgbnQAk8YbtHRyMcU0v2O7RdRhPhlT-vS_q40a7Q6g,50
-dsmq/client.py,sha256=ZsbuvsWX6_4-tkp7i3ixYAPEyvoebKX8qFtW38DHdXE,2569
-dsmq/demo.py,sha256=K53cC5kN7K4kNJlPq7c5OTIMHRCKTo9hYX2aIos57rU,542
-dsmq/example_get_client.py,sha256=PvAsDGEAH1kVBifLVg2rx8ZxnAZmvzVCvZq13VgpLds,301
-dsmq/example_put_client.py,sha256=QxDc3i7KAjjhpwxRRpI0Ke5KTNSPuBf9kkcGyTvUEaw,353
-dsmq/server.py,sha256=HXPH9-nSzz2Iy2UzIWqgUVbL174jq1VBzs7im9vRLqk,8268
-dsmq/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dsmq/tests/integration_test.py,sha256=dLsQGCmpXv4zRb93TriccH7TbUyD9MHcLckAQqfDOK4,5980
-dsmq/tests/performance_suite.py,sha256=E59zB2ZvM8V5f8RxaB7p-Kehqyhrgsl0sXuy7g74BaI,5218
-dsmq-1.3.1.dist-info/METADATA,sha256=K6fqrudTKsXR2xlHDlTNkrjepNcsKn5kX9I5yjpzxPk,4859
-dsmq-1.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dsmq-1.3.1.dist-info/licenses/LICENSE,sha256=3Yu1mAp5VsKmnDtzkiOY7BdmrLeNwwZ3t6iWaLnlL0Y,1071
-dsmq-1.3.1.dist-info/RECORD,,