dsmq 1.3.0__py3-none-any.whl → 1.3.1__py3-none-any.whl

dsmq/client.py

@@ -45,6 +45,7 @@ class DSMQClientSideConnection:
             msg_text = self.websocket.recv()
         except ConnectionClosedError:
             self.close()
+            return ""
 
         msg = json.loads(msg_text)
         return msg["message"]

dsmq/server.py

@@ -11,46 +11,41 @@ _default_host = "127.0.0.1"
 _default_port = 30008
 _n_retries = 20
 _first_retry = 0.005  # seconds
-_time_to_live = 60.0  # seconds
-
-# _db_name = ":memory:"
-_db_name = "file::memory:?cache=shared"
-# May occasionally create files with this name.
-# https://sqlite.org/inmemorydb.html
-# "...parts of a temporary database might be flushed to disk if the
-# database becomes large or if SQLite comes under memory pressure."
+_time_between_cleanup = 0.05  # seconds
+_max_queue_length = 10
 
 # Make this global so it's easy to share
 dsmq_server = None
 
 
-def serve(host=_default_host, port=_default_port, verbose=False):
+def serve(
+    host=_default_host,
+    port=_default_port,
+    name="mqdb",
+    verbose=False,
+):
     """
     For best results, start this running in its own process and walk away.
     """
-    # Cleanup temp files.
-    # Under some condition
-    # (which I haven't yet been able to pin down)
-    # a file is generated with the db name.
-    # If it is not removed, it gets
-    # treated as a SQLite db on disk,
-    # which dramatically slows it down,
-    # especially the way it's used here for
-    # rapid-fire one-item reads and writes.
-    filenames = os.listdir()
-    for filename in filenames:
-        if filename[: len(_db_name)] == _db_name:
-            os.remove(filename)
-
+    # May occasionally create files with this name.
+    # https://sqlite.org/inmemorydb.html
+    # "...parts of a temporary database might be flushed to disk if the
+    # database becomes large or if SQLite comes under memory pressure."
+    global _db_name
+    _db_name = f"file:{name}?mode=memory&cache=shared"
+
+    cleanup_temp_files()
     sqlite_conn = sqlite3.connect(_db_name)
     cursor = sqlite_conn.cursor()
 
     # Tweak the connection to make it faster
     # and keep long-term latency more predictable.
-    # cursor.execute("PRAGMA journal_mode = OFF")
-    cursor.execute("PRAGMA journal_mode = WAL")
-    # cursor.execute("PRAGMA synchronous = OFF")
-    # cursor.execute("PRAGMA secure_delete = OFF")
+    # These also make it more susceptible to corruption during shutdown,
+    # but since dsmq is meant to be ephemeral, that's not a concern.
+    cursor.execute("PRAGMA journal_mode = OFF")
+    cursor.execute("PRAGMA synchronous = OFF")
+    cursor.execute("PRAGMA secure_delete = OFF")
+    cursor.execute("PRAGMA temp_store = MEMORY")
 
     cursor.execute("""
 CREATE TABLE IF NOT EXISTS messages (timestamp DOUBLE, topic TEXT, message TEXT)
@@ -58,12 +53,9 @@ CREATE TABLE IF NOT EXISTS messages (timestamp DOUBLE, topic TEXT, message TEXT)
 
     # Making this global in scope is a way to make it available
     # to the shutdown operation. It's an awkward construction,
-    # and a method of last resort. (If you stumble across this and
-    # figure out something more elegant, please submit a PR!
-    # or send it to me at brohrer@gmail.com,
+    # and a method of last resort.
     global dsmq_server
 
-    # dsmq_server = ws_serve(request_handler, host, port)
     for i_retry in range(_n_retries):
         try:
             with ws_serve(request_handler, host, port) as dsmq_server:
@@ -93,9 +85,31 @@ CREATE TABLE IF NOT EXISTS messages (timestamp DOUBLE, topic TEXT, message TEXT)
             time.sleep(wait_time)
 
     sqlite_conn.close()
+    time.sleep(_time_between_cleanup)
+    cleanup_temp_files()
+
+
+def cleanup_temp_files():
+    # Under some condition
+    # (which I haven't yet been able to pin down)
+    # a file is generated with the db name.
+    # If it is not removed, it gets
+    # treated as a SQLite db on disk,
+    # which dramatically slows it down,
+    # especially the way it's used here for
+    # rapid-fire one-item reads and writes.
+    global _db_name
+    filenames = os.listdir()
+    for filename in filenames:
+        if filename[: len(_db_name)] == _db_name:
+            try:
+                os.remove(filename)
+            except FileNotFoundError:
+                pass
 
 
 def request_handler(websocket):
+    global _db_name
     sqlite_conn = sqlite3.connect(_db_name)
     cursor = sqlite_conn.cursor()
 
@@ -112,23 +126,17 @@ def request_handler(websocket):
             if msg["action"] == "put":
                 msg["timestamp"] = timestamp
 
-                # This block allows for multiple retries if the database
-                # is busy.
-                for i_retry in range(_n_retries):
-                    try:
-                        cursor.execute(
-                            """
-                            INSERT INTO messages (timestamp, topic, message)
-                            VALUES (:timestamp, :topic, :message)
-                            """,
-                            (msg),
-                        )
-                        sqlite_conn.commit()
-                    except sqlite3.OperationalError:
-                        wait_time = _first_retry * 2**i_retry
-                        time.sleep(wait_time)
-                        continue
-                    break
+                try:
+                    cursor.execute(
+                        """
+                        INSERT INTO messages (timestamp, topic, message)
+                        VALUES (:timestamp, :topic, :message)
+                        """,
+                        (msg),
+                    )
+                    sqlite_conn.commit()
+                except sqlite3.OperationalError:
+                    pass
 
             elif msg["action"] == "get":
                 try:
@@ -138,31 +146,25 @@ def request_handler(websocket):
                     last_read_time = last_read_times[topic]
                 msg["last_read_time"] = last_read_time
 
-                # This block allows for multiple retries if the database
-                # is busy.
-                for i_retry in range(_n_retries):
-                    try:
-                        cursor.execute(
-                            """
-                            SELECT message,
-                            timestamp
-                            FROM messages,
-                            (
-                            SELECT MIN(timestamp) AS min_time
-                            FROM messages
-                            WHERE topic = :topic
-                            AND timestamp > :last_read_time
-                            ) a
-                            WHERE topic = :topic
-                            AND timestamp = a.min_time
-                            """,
-                            msg,
-                        )
-                    except sqlite3.OperationalError:
-                        wait_time = _first_retry * 2**i_retry
-                        time.sleep(wait_time)
-                        continue
-                    break
+                try:
+                    cursor.execute(
+                        """
+                        SELECT message,
+                        timestamp
+                        FROM messages,
+                        (
+                        SELECT MIN(timestamp) AS min_time
+                        FROM messages
+                        WHERE topic = :topic
+                        AND timestamp > :last_read_time
+                        ) a
+                        WHERE topic = :topic
+                        AND timestamp = a.min_time
+                        """,
+                        msg,
+                    )
+                except sqlite3.OperationalError:
+                    pass
 
                 try:
                     result = cursor.fetchall()[0]
@@ -174,20 +176,14 @@ def request_handler(websocket):
                     message = ""
 
                 websocket.send(json.dumps({"message": message}))
+
             elif msg["action"] == "shutdown":
                 # Run this from a separate thread to prevent deadlock
                 global dsmq_server
 
                 def shutdown_gracefully(server_to_shutdown):
                     server_to_shutdown.shutdown()
-
-                    filenames = os.listdir()
-                    for filename in filenames:
-                        if filename[: len(_db_name)] == _db_name:
-                            try:
-                                os.remove(filename)
-                            except FileNotFoundError:
-                                pass
+                    # cleanup_temp_files()
 
                 Thread(target=shutdown_gracefully, args=(dsmq_server,)).start()
                 break
@@ -196,23 +192,37 @@ def request_handler(websocket):
                     "dsmq client action must either be 'put', 'get', or 'shutdown'"
                 )
 
-            # Periodically clean out messages from the queue that are
-            # past their sell buy date.
-            # This operation is pretty fast. I clock it at 12 us on my machine.
-            if time.time() - time_of_last_purge > _time_to_live:
+            # Periodically clean out messages to keep individual queues at
+            # a manageable length and the overall mq small.
+            if time.time() - time_of_last_purge > _time_between_cleanup:
                 try:
                     cursor.execute(
                         """
-                        DELETE FROM messages
-                        WHERE timestamp < :time_threshold
+                        DELETE
+                        FROM messages
+                        WHERE topic = :topic
+                        AND timestamp IN (
+                          SELECT timestamp
+                          FROM (
+                              SELECT timestamp,
+                              RANK() OVER (ORDER BY timestamp DESC) recency_rank
+                              FROM messages
+                              WHERE topic = :topic
+                          )
+                          WHERE recency_rank >= :max_queue_length
+                        )
                         """,
-                        {"time_threshold": time_of_last_purge},
+                        {
+                            "max_queue_length": _max_queue_length,
+                            "topic": topic,
+                        },
                     )
                     sqlite_conn.commit()
                     time_of_last_purge = time.time()
                 except sqlite3.OperationalError:
                     # Database may be locked. Try again next time.
                     pass
+
     except (ConnectionClosedError, ConnectionClosedOK):
         # Something happened on the other end and this handler
         # is no longer needed.

{dsmq-1.3.0.dist-info → dsmq-1.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsmq
-Version: 1.3.0
+Version: 1.3.1
 Summary: A dead simple message queue
 License-File: LICENSE
 Requires-Python: >=3.10

{dsmq-1.3.0.dist-info → dsmq-1.3.1.dist-info}/RECORD

@@ -1,15 +1,13 @@
-dsmq/.server.py.swp,sha256=RLjzmw9FyOQY73HurM9PEjp4meAgVEZIN8M92OqgBzw,28672
 dsmq/__init__.py,sha256=YCgbnQAk8YbtHRyMcU0v2O7RdRhPhlT-vS_q40a7Q6g,50
-dsmq/client.py,sha256=_JJlDFTw1-VsRNsm-grYvexrWkkh50n62Q6-aGWum5Q,2547
+dsmq/client.py,sha256=ZsbuvsWX6_4-tkp7i3ixYAPEyvoebKX8qFtW38DHdXE,2569
 dsmq/demo.py,sha256=K53cC5kN7K4kNJlPq7c5OTIMHRCKTo9hYX2aIos57rU,542
 dsmq/example_get_client.py,sha256=PvAsDGEAH1kVBifLVg2rx8ZxnAZmvzVCvZq13VgpLds,301
 dsmq/example_put_client.py,sha256=QxDc3i7KAjjhpwxRRpI0Ke5KTNSPuBf9kkcGyTvUEaw,353
-dsmq/server.py,sha256=Ej6iQ1aw4LVHyHvdeZxxwSZS3VBIQP0qsFstuo0hwnY,8487
-dsmq/tests/.performance_suite.py.swp,sha256=D3B86JpgBIYDE0at6nG2Uw9WWASFxp12mB9zlNXYPbA,24576
+dsmq/server.py,sha256=HXPH9-nSzz2Iy2UzIWqgUVbL174jq1VBzs7im9vRLqk,8268
 dsmq/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dsmq/tests/integration_test.py,sha256=dLsQGCmpXv4zRb93TriccH7TbUyD9MHcLckAQqfDOK4,5980
 dsmq/tests/performance_suite.py,sha256=E59zB2ZvM8V5f8RxaB7p-Kehqyhrgsl0sXuy7g74BaI,5218
-dsmq-1.3.0.dist-info/METADATA,sha256=bZRJ0Oz5Vnavl4AU9bO2f7TuktSpUOJQ9hyx7Yusne4,4859
-dsmq-1.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dsmq-1.3.0.dist-info/licenses/LICENSE,sha256=3Yu1mAp5VsKmnDtzkiOY7BdmrLeNwwZ3t6iWaLnlL0Y,1071
-dsmq-1.3.0.dist-info/RECORD,,
+dsmq-1.3.1.dist-info/METADATA,sha256=K6fqrudTKsXR2xlHDlTNkrjepNcsKn5kX9I5yjpzxPk,4859
+dsmq-1.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dsmq-1.3.1.dist-info/licenses/LICENSE,sha256=3Yu1mAp5VsKmnDtzkiOY7BdmrLeNwwZ3t6iWaLnlL0Y,1071
+dsmq-1.3.1.dist-info/RECORD,,