dsmq 1.2.4__py3-none-any.whl → 1.3.0__py3-none-any.whl

dsmq/client.py

@@ -8,6 +8,7 @@ _default_port = 30008
 
 _n_retries = 10
 _initial_retry = 0.01  # seconds
+_shutdown_delay = 0.1  # seconds
 
 
 def connect(host=_default_host, port=_default_port, verbose=False):
@@ -68,6 +69,10 @@ class DSMQClientSideConnection:
     def shutdown_server(self):
         msg_dict = {"action": "shutdown", "topic": ""}
         self.websocket.send(json.dumps(msg_dict))
+        # Give the server time to wind down
+        time.sleep(_shutdown_delay)
 
     def close(self):
         self.websocket.close()
+        # Give the websocket time to wind down
+        time.sleep(_shutdown_delay)

dsmq/server.py

@@ -9,12 +9,16 @@ from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK
 
 _default_host = "127.0.0.1"
 _default_port = 30008
-_n_retries = 5
-_first_retry = 0.01  # seconds
-_pause = 0.01  # seconds
-_time_to_live = 600.0  # seconds
+_n_retries = 20
+_first_retry = 0.005  # seconds
+_time_to_live = 60.0  # seconds
 
+# _db_name = ":memory:"
 _db_name = "file::memory:?cache=shared"
+# May occasionally create files with this name.
+# https://sqlite.org/inmemorydb.html
+# "...parts of a temporary database might be flushed to disk if the
+# database becomes large or if SQLite comes under memory pressure."
 
 # Make this global so it's easy to share
 dsmq_server = None
@@ -40,6 +44,14 @@ def serve(host=_default_host, port=_default_port, verbose=False):
 
     sqlite_conn = sqlite3.connect(_db_name)
     cursor = sqlite_conn.cursor()
+
+    # Tweak the connection to make it faster
+    # and keep long-term latency more predictable.
+    # cursor.execute("PRAGMA journal_mode = OFF")
+    cursor.execute("PRAGMA journal_mode = WAL")
+    # cursor.execute("PRAGMA synchronous = OFF")
+    # cursor.execute("PRAGMA secure_delete = OFF")
+
     cursor.execute("""
 CREATE TABLE IF NOT EXISTS messages (timestamp DOUBLE, topic TEXT, message TEXT)
     """)
@@ -52,12 +64,33 @@ CREATE TABLE IF NOT EXISTS messages (timestamp DOUBLE, topic TEXT, message TEXT)
     global dsmq_server
 
     # dsmq_server = ws_serve(request_handler, host, port)
-    with ws_serve(request_handler, host, port) as dsmq_server:
-        dsmq_server.serve_forever()
-    if verbose:
-        print()
-        print(f"Server started at {host} on port {port}.")
-        print("Waiting for clients...")
+    for i_retry in range(_n_retries):
+        try:
+            with ws_serve(request_handler, host, port) as dsmq_server:
+                dsmq_server.serve_forever()
+
+            if verbose:
+                print()
+                print(f"Server started at {host} on port {port}.")
+                print("Waiting for clients...")
+
+            break
+
+        except OSError:
+            # Catch the case where the address is already in use
+            if verbose:
+                print()
+                if i_retry < _n_retries - 1:
+                    print(f"Couldn't start dsmq server on {host} on port {port}.")
+                    print(f"    Trying again ({i_retry}) ...")
+                else:
+                    print()
+                    print(f"Failed to start dsmq server on {host} on port {port}.")
+                    print()
+                    raise
+
+            wait_time = _first_retry * 2**i_retry
+            time.sleep(wait_time)
 
     sqlite_conn.close()
 

dsmq/tests/performance_suite.py

@@ -0,0 +1,179 @@
+import multiprocessing as mp
+import time
+
+from dsmq.server import serve
+from dsmq.client import connect
+
+host = "127.0.0.1"
+port = 30303
+verbose = False
+
+_pause = 0.01
+_very_long_pause = 1.0
+
+_n_iter = int(1e3)
+_n_long_char = int(1e4)
+
+_short_msg = "q"
+_long_msg = str(["q"] * _n_long_char)
+
+_test_topic = "test"
+
+
+def main():
+    print()
+    print("dsmq timing measurements")
+
+    time_short_writes()
+    time_long_writes()
+    time_empty_reads()
+    time_short_reads()
+    time_long_reads()
+
+
+def time_short_writes():
+    condition = "short write"
+
+    duration, duration_close = time_writes(msg=_short_msg, n_iter=1)
+
+    print()
+    print(f"Time for first {condition}  [including closing]")
+    print(f"        {int(duration)} μs  [{int(duration_close)} μs]")
+
+    avg_duration, avg_duration_close = time_writes(msg=_short_msg, n_iter=_n_iter)
+
+    print(f"Average time for a {condition}  [including closing]")
+    print(f"        {int(avg_duration)} μs  [{int(avg_duration_close)} μs]")
+
+
+def time_long_writes():
+    duration, duration_close = time_writes(msg=_long_msg, n_iter=1)
+
+    condition = "long write"
+    print()
+    print(f"Time for first {condition}  [including closing]")
+    print(f"        {int(duration)} μs  [{int(duration_close)} μs]")
+
+    avg_duration, avg_duration_close = time_writes(msg=_long_msg, n_iter=_n_iter)
+
+    condition = f"long write ({_n_long_char} characters)"
+    print(f"Average time for a {condition}  [including closing]")
+    print(f"        {int(avg_duration)} μs  [{int(avg_duration_close)} μs]")
+
+    condition = "long write (per 1000 characters)"
+    print(f"Average time for a {condition}  [including closing]")
+    print(
+        f"        {int(1000 * avg_duration / _n_long_char)} μs  "
+        + f"[{int(1000 * avg_duration_close / _n_long_char)}] μs"
+    )
+
+
+def time_writes(msg="message", n_iter=1):
+    p_server = mp.Process(target=serve, args=(host, port, verbose))
+    p_server.start()
+    time.sleep(_pause)
+    write_client = connect(host, port)
+
+    start_time = time.time()
+    for _ in range(n_iter):
+        write_client.put(_test_topic, msg)
+    avg_duration = 1e6 * (time.time() - start_time) / n_iter  # microseconds
+
+    write_client.shutdown_server()
+    write_client.close()
+
+    p_server.join(_very_long_pause)
+    if p_server.is_alive():
+        print("    Doing a hard shutdown on mq server")
+        p_server.kill()
+    avg_duration_close = 1e6 * (time.time() - start_time) / n_iter  # microseconds
+
+    return avg_duration, avg_duration_close
+
+
+def time_empty_reads():
+    condition = "empty read"
+
+    duration, duration_close = time_reads(msg=None, n_iter=1)
+
+    print()
+    print(f"Time for first {condition}  [including closing]")
+    print(f"        {int(duration)} μs  [{int(duration_close)} μs]")
+
+    avg_duration, avg_duration_close = time_reads(msg=None, n_iter=_n_iter)
+
+    print(f"Average time for a {condition}  [including closing]")
+    print(f"        {int(avg_duration)} μs  [{int(avg_duration_close)} μs]")
+
+
+def time_short_reads():
+    condition = "short read"
+
+    duration, duration_close = time_reads(msg=_short_msg, n_iter=1)
+
+    print()
+    print(f"Time for first {condition}  [including closing]")
+    print(f"        {int(duration)} μs  [{int(duration_close)} μs]")
+
+    avg_duration, avg_duration_close = time_reads(msg=_short_msg, n_iter=_n_iter)
+
+    print(f"Average time for a {condition}  [including closing]")
+    print(f"        {int(avg_duration)} μs  [{int(avg_duration_close)} μs]")
+
+
+def time_long_reads():
+    condition = f"long read ({_n_long_char} characters)"
+
+    duration, duration_close = time_reads(msg=_long_msg, n_iter=1)
+
+    print()
+    print(f"Time for first {condition}  [including closing]")
+    print(f"        {int(duration)} μs  [{int(duration_close)} μs]")
+
+    avg_duration, avg_duration_close = time_reads(msg=_long_msg, n_iter=_n_iter)
+
+    print(f"Average time for a {condition}  [including closing]")
+    print(f"        {int(avg_duration)} μs  [{int(avg_duration_close)} μs]")
+
+    condition = "long read (per 1000 characters)"
+    print(f"Average time for a {condition}  [including closing]")
+    print(
+        f"        {int(1000 * avg_duration / _n_long_char)} μs  "
+        + f"[{int(1000 * avg_duration_close / _n_long_char)}] μs"
+    )
+
+
+def time_reads(msg=None, n_iter=1):
+    p_server = mp.Process(target=serve, args=(host, port, verbose))
+    p_server.start()
+    time.sleep(_pause)
+    # write_client = connect(host, port)
+    read_client = connect(host, port)
+
+    if msg is not None:
+        for _ in range(n_iter):
+            read_client.put(_test_topic, msg)
+
+    start_time = time.time()
+    for _ in range(n_iter):
+        msg = read_client.get(_test_topic)
+
+    avg_duration = 1e6 * (time.time() - start_time) / n_iter  # microseconds
+
+    read_client.shutdown_server()
+    # write_client.close()
+    read_client.close()
+
+    p_server.join(_very_long_pause)
+
+    if p_server.is_alive():
+        print("    Doing a hard shutdown on mq server")
+        p_server.kill()
+
+    avg_duration_close = 1e6 * (time.time() - start_time) / n_iter  # microseconds
+
+    return avg_duration, avg_duration_close
+
+
+if __name__ == "__main__":
+    main()

{dsmq-1.2.4.dist-info → dsmq-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsmq
-Version: 1.2.4
+Version: 1.3.0
 Summary: A dead simple message queue
 License-File: LICENSE
 Requires-Python: >=3.10
@@ -178,3 +178,8 @@ Run all the tests in `src/dsmq/tests/` with pytest, for example
 ```
 uv run pytest
 ```
+
+# Performance characterization
+
+Time typical operations on your system with the script at
+`src/dsmq/tests/performance_suite.py`

dsmq-1.3.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+dsmq/.server.py.swp,sha256=RLjzmw9FyOQY73HurM9PEjp4meAgVEZIN8M92OqgBzw,28672
+dsmq/__init__.py,sha256=YCgbnQAk8YbtHRyMcU0v2O7RdRhPhlT-vS_q40a7Q6g,50
+dsmq/client.py,sha256=_JJlDFTw1-VsRNsm-grYvexrWkkh50n62Q6-aGWum5Q,2547
+dsmq/demo.py,sha256=K53cC5kN7K4kNJlPq7c5OTIMHRCKTo9hYX2aIos57rU,542
+dsmq/example_get_client.py,sha256=PvAsDGEAH1kVBifLVg2rx8ZxnAZmvzVCvZq13VgpLds,301
+dsmq/example_put_client.py,sha256=QxDc3i7KAjjhpwxRRpI0Ke5KTNSPuBf9kkcGyTvUEaw,353
+dsmq/server.py,sha256=Ej6iQ1aw4LVHyHvdeZxxwSZS3VBIQP0qsFstuo0hwnY,8487
+dsmq/tests/.performance_suite.py.swp,sha256=D3B86JpgBIYDE0at6nG2Uw9WWASFxp12mB9zlNXYPbA,24576
+dsmq/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dsmq/tests/integration_test.py,sha256=dLsQGCmpXv4zRb93TriccH7TbUyD9MHcLckAQqfDOK4,5980
+dsmq/tests/performance_suite.py,sha256=E59zB2ZvM8V5f8RxaB7p-Kehqyhrgsl0sXuy7g74BaI,5218
+dsmq-1.3.0.dist-info/METADATA,sha256=bZRJ0Oz5Vnavl4AU9bO2f7TuktSpUOJQ9hyx7Yusne4,4859
+dsmq-1.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dsmq-1.3.0.dist-info/licenses/LICENSE,sha256=3Yu1mAp5VsKmnDtzkiOY7BdmrLeNwwZ3t6iWaLnlL0Y,1071
+dsmq-1.3.0.dist-info/RECORD,,

dsmq-1.2.4.dist-info/RECORD

@@ -1,12 +0,0 @@
-dsmq/__init__.py,sha256=YCgbnQAk8YbtHRyMcU0v2O7RdRhPhlT-vS_q40a7Q6g,50
-dsmq/client.py,sha256=p6irQZOE4b2fpTUwSrEUraPmrJzvT8QSU01ak9qpGCQ,2351
-dsmq/demo.py,sha256=K53cC5kN7K4kNJlPq7c5OTIMHRCKTo9hYX2aIos57rU,542
-dsmq/example_get_client.py,sha256=PvAsDGEAH1kVBifLVg2rx8ZxnAZmvzVCvZq13VgpLds,301
-dsmq/example_put_client.py,sha256=QxDc3i7KAjjhpwxRRpI0Ke5KTNSPuBf9kkcGyTvUEaw,353
-dsmq/server.py,sha256=5c4mHWz6yslSxpOwrZ3vOCMcx_cm8zMF8iJNnQF8ft0,7249
-dsmq/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dsmq/tests/integration_test.py,sha256=dLsQGCmpXv4zRb93TriccH7TbUyD9MHcLckAQqfDOK4,5980
-dsmq-1.2.4.dist-info/METADATA,sha256=CswX8bQph2pF-k-VMCLAPqaho3Mhny4xDLJ70xwWa9I,4730
-dsmq-1.2.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dsmq-1.2.4.dist-info/licenses/LICENSE,sha256=3Yu1mAp5VsKmnDtzkiOY7BdmrLeNwwZ3t6iWaLnlL0Y,1071
-dsmq-1.2.4.dist-info/RECORD,,