PgQueuer 0.2.2__tar.gz → 0.2.4__tar.gz

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PgQueuer
-Version: 0.2.2
+Version: 0.2.4
 Summary: PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing.
 Author: janbjorge
 License: MIT License
@@ -30,6 +30,7 @@ Requires-Dist: anyio>=4.0
 Requires-Dist: asyncpg>=0.27.0
 Requires-Dist: pgcachewatch>=0.4
 Requires-Dist: pydantic>=2.0.0
+Requires-Dist: tabulate>=0.9.0
 Provides-Extra: dev
 Requires-Dist: asyncpg-stubs; extra == "dev"
 Requires-Dist: mypy-extensions; extra == "dev"
@@ -37,6 +38,7 @@ Requires-Dist: mypy; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
+Requires-Dist: types-tabulate; extra == "dev"
 
 ##  🚀 PgQueuer - Building Smoother Workflows One Queue at a Time 🚀
 [![CI](https://github.com/janbjorge/PgQueuer/actions/workflows/ci.yml/badge.svg)](https://github.com/janbjorge/PgQueuer/actions/workflows/ci.yml?query=branch%3Amain)

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
     "asyncpg>=0.27.0",
     "pgcachewatch>=0.4",
     "pydantic>=2.0.0",
+    "tabulate>=0.9.0",
 ]
 
 [project.urls]
@@ -50,6 +51,7 @@ dev = [
     "pytest-asyncio",
     "pytest",
     "ruff",
+    "types-tabulate",
 ]
 
 [tool.setuptools_scm]

pgqueuer-0.2.4/src/PgQueuer/__main__.py

@@ -0,0 +1,8 @@
+import asyncio
+import contextlib
+
+from PgQueuer import cli
+
+if __name__ == "__main__":
+    with contextlib.suppress(KeyboardInterrupt):
+        asyncio.run(cli.main())

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/src/PgQueuer/cli.py

@@ -1,9 +1,59 @@
 import argparse
+import asyncio
 import os
+from datetime import timedelta
 
 import asyncpg
+from tabulate import tabulate, tabulate_formats
+
+from PgQueuer.models import LogStatistics
+from PgQueuer.queries import Queries
+
+
+async def display_stats(
+    log_stats: list[LogStatistics],
+    tablefmt: str,
+) -> None:
+    print(
+        tabulate(
+            [
+                (
+                    stat.created,
+                    stat.count,
+                    stat.entrypoint,
+                    stat.time_in_queue,
+                    stat.status,
+                    stat.priority,
+                )
+                for stat in log_stats
+            ],
+            headers=[
+                "Created",
+                "Count",
+                "Entrypoint",
+                "Time in Queue (HH:MM:SS)",
+                "Status",
+                "Priority",
+            ],
+            tablefmt=tablefmt,
+        )
+    )
+
 
-from PgQueuer import queries
+async def fetch_and_dispay(
+    queries: Queries,
+    interval: timedelta | None,
+    tail: int,
+    tablefmt: str,
+) -> None:
+    clear_and_home = "\033[2J\033[H"
+    while True:
+        print(clear_and_home, end="")
+        stats = await queries.log_statistics(tail)  # Fetch stats once and reuse
+        await display_stats(stats, tablefmt)
+        if interval is None:
+            return
+        await asyncio.sleep(interval.total_seconds())
 
 
 def cliparser() -> argparse.Namespace:
@@ -100,6 +150,35 @@ def cliparser() -> argparse.Namespace:
         parents=[common_arguments],
     )
 
+    dashboardparser = subparsers.add_parser(
+        "dashboard",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=[common_arguments],
+    )
+    dashboardparser.add_argument(
+        "-i",
+        "--interval",
+        help="Set the refresh interval in seconds for updating the dashboard display.",
+        default=None,
+        type=lambda x: timedelta(seconds=float(x)),
+    )
+    dashboardparser.add_argument(
+        "-n",
+        "--tail",
+        help="The number of the most recent log entries to display on the dashboard.",
+        type=int,
+        default=25,
+    )
+    dashboardparser.add_argument(
+        "--table-format",
+        default="pretty",
+        help=(
+            "Specify the format of the table used to display statistics. "
+            "Options are provided by the tabulate library."
+        ),
+        choices=tabulate_formats,
+    )
+
     return parser.parse_args()
 
 
@@ -120,9 +199,19 @@ async def main() -> None:
         port=parsed.pg_port,
         user=parsed.pg_user,
         host=parsed.pg_host,
+        max_size=1,
+        min_size=0,
     ) as pool:
+        queries = Queries(pool)
         match parsed.command:
             case "install":
-                await queries.Queries(pool=pool).install()
+                await queries.install()
             case "uninstall":
-                await queries.Queries(pool=pool).uninstall()
+                await queries.uninstall()
+            case "dashboard":
+                await fetch_and_dispay(
+                    queries,
+                    parsed.interval,
+                    parsed.tail,
+                    parsed.table_format,
+                )

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/src/PgQueuer/models.py

@@ -1,3 +1,4 @@
+from datetime import timedelta
 from typing import Literal
 
 from pydantic import AwareDatetime, BaseModel
@@ -26,7 +27,7 @@ class Job(BaseModel):
     payload: bytes | None
 
 
-class QueueSize(BaseModel):
+class QueueStatistics(BaseModel):
     """
     Represents the number of jobs per entrypoint and priority in the queue.
     """
@@ -36,12 +37,14 @@ class QueueSize(BaseModel):
     priority: int
 
 
-class LogSize(BaseModel):
+class LogStatistics(BaseModel):
     """
-    Represents log details for jobs based on status, entrypoint, and priority.
+    Represents log statistics for jobs based on status, entrypoint, and priority.
     """
 
     count: int
+    created: AwareDatetime
     entrypoint: str
     priority: int
     status: STATUS_LOG
+    time_in_queue: timedelta

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/src/PgQueuer/qm.py

@@ -41,8 +41,19 @@ def is_async_callable(obj: SyncEntrypoint) -> TypeGuard[SyncEntrypoint]: ...
 
 
 def is_async_callable(obj: object) -> bool:
-    # Inspired by:
-    # https://github.com/encode/starlette/blob/9f16bf5c25e126200701f6e04330864f4a91a898/starlette/_utils.py#L38
+    """
+    Determines whether an object is an asynchronous callable.
+    This function identifies objects that are either asynchronous coroutine functions
+    or have a callable __call__ method that is an asynchronous coroutine function.
+
+    The function handles functools.partial objects by evaluating the
+    underlying function. It supports overloads to ensure type-specific behavior
+    with AsyncEntrypoint and SyncEntrypoint types.
+
+    Inspired by:
+    https://github.com/encode/starlette/blob/9f16bf5c25e126200701f6e04330864f4a91a898/starlette/_utils.py#L38
+    """
+
     while isinstance(obj, functools.partial):
         obj = obj.func
 
@@ -60,17 +71,12 @@ class QueueManager:
 
     pool: asyncpg.Pool
     queries: Queries = dataclasses.field(init=False)
-    channel: PGChannel = dataclasses.field(
-        default=PGChannel(DBSettings().channel),
-    )
-    alive: bool = dataclasses.field(
-        init=False,
-        default=True,
-    )
+    channel: PGChannel = dataclasses.field(default=PGChannel(DBSettings().channel))
+    alive: bool = dataclasses.field(init=False, default=True)
+
     # Should registry be a weakref?
     registry: dict[str, Entrypoint] = dataclasses.field(
-        init=False,
-        default_factory=dict,
+        init=False, default_factory=dict
     )
 
     def __post_init__(self) -> None:
@@ -84,8 +90,8 @@ class QueueManager:
 
     def entrypoint(self, name: str) -> Callable[[T], T]:
         """
-        Decorator to register a function as an entrypoint for
-        handling specific job types.
+        Registers a function as an entrypoint for handling specific
+        job types. Ensures unique naming in the registry.
         """
 
         if name in self.registry:
@@ -102,8 +108,8 @@ class QueueManager:
         dequeue_timeout: timedelta = timedelta(seconds=30),
     ) -> None:
         """
-        Starts the event listener and continuously dispatches jobs to
-        registered entry points until stopped.
+        Continuously listens for events and dispatches jobs. Manages connections and
+        tasks, logs timeouts, and resets connections upon termination.
         """
         async with (
             self.pool.acquire() as conn,
@@ -131,8 +137,8 @@ class QueueManager:
 
     async def _dispatch(self, job: Job) -> None:
         """
-        Internal method to asynchronously handle job dispatch,
-        including exception logging and job status updates.
+        Handles asynchronous job dispatch. Logs exceptions, updates job status,
+        and adapts execution method based on whether the job's function is asynchronous.
         """
 
         logger.debug(

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/src/PgQueuer/queries.py

@@ -31,12 +31,11 @@ class DBSettings:
         default_factory=lambda: add_prefix("fn_pgqueuer_changed"),
         kw_only=True,
     )
-    log_table: Final[str] = dataclasses.field(
-        default_factory=lambda: add_prefix("pgqueuer_log"),
-        kw_only=True,
+    statistics_table: Final[str] = dataclasses.field(
+        default_factory=lambda: add_prefix("pgqueuer_statistics"),
     )
-    log_table_status_type: Final[str] = dataclasses.field(
-        default_factory=lambda: add_prefix("pgqueuer_status_log"),
+    statistics_table_status_type: Final[str] = dataclasses.field(
+        default_factory=lambda: add_prefix("pgqueuer_statistics_status"),
         kw_only=True,
     )
     queue_status_type: Final[str] = dataclasses.field(
@@ -74,7 +73,7 @@ class Queries:
             CREATE TABLE {self.settings.queue_table} (
                 id SERIAL PRIMARY KEY,
                 priority INT NOT NULL,
-                created TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                created TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
                 status {self.settings.queue_status_type} NOT NULL,
                 entrypoint TEXT NOT NULL,
                 payload BYTEA
@@ -82,15 +81,23 @@ class Queries:
             CREATE INDEX ON {self.settings.queue_table} (priority ASC, id DESC)
                 INCLUDE (id) WHERE status = 'queued';
 
-            CREATE TYPE {self.settings.log_table_status_type} AS ENUM ('exception', 'successful');
-            CREATE TABLE {self.settings.log_table} (
+            CREATE TYPE {self.settings.statistics_table_status_type} AS ENUM ('exception', 'successful');
+            CREATE TABLE {self.settings.statistics_table} (
                 id SERIAL PRIMARY KEY,
+                created TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT DATE_TRUNC('sec', CURRENT_TIMESTAMP at time zone 'UTC'),
+                count BIGINT NOT NULL,
                 priority INT NOT NULL,
-                created TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
-                duration INTERVAL NOT NULL,
-                status {self.settings.log_table_status_type} NOT NULL,
+                time_in_queue INTERVAL NOT NULL,
+                status {self.settings.statistics_table_status_type} NOT NULL,
                 entrypoint TEXT NOT NULL
             );
+            CREATE UNIQUE INDEX unique_count ON {self.settings.statistics_table} (
+                priority,
+                DATE_TRUNC('sec', created at time zone 'UTC'),
+                DATE_TRUNC('sec', time_in_queue),
+                status,
+                entrypoint
+            );
 
             CREATE FUNCTION {self.settings.function}() RETURNS TRIGGER AS $$
             DECLARE
@@ -148,9 +155,9 @@ class Queries:
             DROP TRIGGER {self.settings.trigger};
             DROP FUNCTION {self.settings.function};
             DROP TABLE {self.settings.queue_table};
-            DROP TABLE {self.settings.log_table};
+            DROP TABLE {self.settings.statistics_table};
             DROP TYPE {self.settings.queue_status_type};
-            DROP TYPE {self.settings.log_table_status_type};
+            DROP TYPE {self.settings.statistics_table_status_type};
         """
         await self.pool.execute(query)
 
@@ -182,7 +189,7 @@ class Queries:
     async def enqueue(
         self,
         entrypoint: str | list[str],
-        payload: bytes | None | list[bytes | None],
+        payload: bytes | None | list[bytes] | list[None] | list[bytes | None],
         priority: int | list[int] = 0,
     ) -> None:
         """
@@ -231,7 +238,7 @@ class Queries:
             query = f"TRUNCATE {self.settings.queue_table}"
             await self.pool.execute(query)
 
-    async def queue_size(self) -> list[models.QueueSize]:
+    async def queue_size(self) -> list[models.QueueStatistics]:
         """
         Returns the number of jobs in the queue grouped by entrypoint and priority.
         """
@@ -242,10 +249,11 @@ class Queries:
                 entrypoint
             FROM
                 {self.settings.queue_table}
-            GROUP BY priority, entrypoint
+            GROUP BY entrypoint, priority
+            ORDER BY count, entrypoint, priority
         """
         return [
-            models.QueueSize.model_validate(dict(x))
+            models.QueueStatistics.model_validate(dict(x))
             for x in await self.pool.fetch(query)
         ]
 
@@ -254,20 +262,56 @@ class Queries:
         Moves a completed or failed job from the queue table to the log
         table, recording its final status and duration.
         """
+
         query = f"""
-            WITH moved_row AS (
+            WITH deleted AS (
                 DELETE FROM {self.settings.queue_table}
                 WHERE id = $1
-                RETURNING id, priority, created, entrypoint
             )
 
-            INSERT INTO {self.settings.log_table} (
-                id, priority, created, duration, status, entrypoint
+            INSERT INTO {self.settings.statistics_table} (
+                priority,
+                entrypoint,
+                time_in_queue,
+                created,
+                status,
+                count
+            )
+            VALUES (
+                $2,                                         -- priority
+                $3,                                         -- entrypoint
+                DATE_TRUNC('sec', NOW() - $4),              -- time_in_queue
+                DATE_TRUNC('sec', $4 at time zone 'UTC'),   -- created at time zone 'UTC'
+                $5,                                         -- status
+                1                                           -- start count
             )
-                SELECT id, priority, created, NOW() - created, $2, entrypoint
-                FROM moved_row;
-         """
-        await self.pool.execute(query, job.id, status)
+            ON CONFLICT (
+                priority,
+                entrypoint,
+                DATE_TRUNC('sec', created at time zone 'UTC'),
+                DATE_TRUNC('sec', time_in_queue),
+                status
+            )
+            DO UPDATE
+                SET
+                    count = {self.settings.statistics_table}.count + 1
+                WHERE
+                        {self.settings.statistics_table}.priority = $2
+                    AND DATE_TRUNC('sec', {self.settings.statistics_table}.time_in_queue) = DATE_TRUNC('sec', NOW() - $4)
+                    AND DATE_TRUNC('sec', {self.settings.statistics_table}.created) = DATE_TRUNC('sec', $4 at time zone 'UTC')
+                    AND {self.settings.statistics_table}.status = $5
+                    AND {self.settings.statistics_table}.entrypoint = $3
+
+         """  # noqa: E501
+
+        await self.pool.execute(
+            query,
+            job.id,
+            job.priority,
+            job.entrypoint,
+            job.created,
+            status,
+        )
 
     async def clear_log(self, entrypoint: str | list[str] | None = None) -> None:
         """
@@ -275,29 +319,35 @@ class Queries:
         by entrypoint if specified.
         """
         if entrypoint:
-            query = f"DELETE FROM {self.settings.log_table} WHERE entrypoint = ANY($1)"
+            query = f"""
+                DELETE FROM {self.settings.statistics_table}
+                WHERE entrypoint = ANY($1)
+            """
             await self.pool.execute(
                 query,
                 [entrypoint] if isinstance(entrypoint, str) else entrypoint,
             )
         else:
-            query = f"TRUNCATE {self.settings.log_table}"
+            query = f"TRUNCATE {self.settings.statistics_table}"
             await self.pool.execute(query)
 
-    async def log_size(self) -> list[models.LogSize]:
-        """
-        Returns the number of log entries grouped by status, entrypoint, and priority.
-        """
+    async def log_statistics(self, tail: int) -> list[models.LogStatistics]:
         query = f"""
             SELECT
-                count(*) AS count,
-                status,
+                count,
+                created,
+                time_in_queue,
+                entrypoint,
                 priority,
-                entrypoint
-            FROM
-                {self.settings.log_table}
-            GROUP BY status, priority, entrypoint
+                status
+            FROM {self.settings.statistics_table}
+            ORDER BY (
+                id, created, count, time_in_queue,
+                entrypoint, priority, status
+            ) DESC
+            LIMIT $1
         """
         return [
-            models.LogSize.model_validate(dict(x)) for x in await self.pool.fetch(query)
+            models.LogStatistics.model_validate(dict(x))
+            for x in await self.pool.fetch(query, tail)
         ]

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PgQueuer
-Version: 0.2.2
+Version: 0.2.4
 Summary: PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing.
 Author: janbjorge
 License: MIT License
@@ -30,6 +30,7 @@ Requires-Dist: anyio>=4.0
 Requires-Dist: asyncpg>=0.27.0
 Requires-Dist: pgcachewatch>=0.4
 Requires-Dist: pydantic>=2.0.0
+Requires-Dist: tabulate>=0.9.0
 Provides-Extra: dev
 Requires-Dist: asyncpg-stubs; extra == "dev"
 Requires-Dist: mypy-extensions; extra == "dev"
@@ -37,6 +38,7 @@ Requires-Dist: mypy; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
+Requires-Dist: types-tabulate; extra == "dev"
 
 ##  🚀 PgQueuer - Building Smoother Workflows One Queue at a Time 🚀
 [![CI](https://github.com/janbjorge/PgQueuer/actions/workflows/ci.yml/badge.svg)](https://github.com/janbjorge/PgQueuer/actions/workflows/ci.yml?query=branch%3Amain)

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/requires.txt

@@ -2,6 +2,7 @@ anyio>=4.0
 asyncpg>=0.27.0
 pgcachewatch>=0.4
 pydantic>=2.0.0
+tabulate>=0.9.0
 
 [dev]
 asyncpg-stubs
@@ -10,3 +11,4 @@ mypy
 pytest-asyncio
 pytest
 ruff
+types-tabulate

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/src/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.2'
-__version_tuple__ = version_tuple = (0, 2, 2)
+__version__ = version = '0.2.4'
+__version_tuple__ = version_tuple = (0, 2, 4)

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/test/test_qm.py

@@ -24,8 +24,11 @@ async def test_job_queing(
         assert context
         seen.append(int(context.payload))
 
-    for n in range(N):
-        await c.queries.enqueue("fetch", f"{n}".encode())
+    await c.queries.enqueue(
+        ["fetch"] * N,
+        [f"{n}".encode() for n in range(N)],
+        [0] * N,
+    )
 
     # Stop flag
     await c.queries.enqueue("fetch", None)
@@ -56,8 +59,11 @@ async def test_job_fetch(
             assert context
             seen.append(int(context.payload))
 
-    for n in range(N):
-        await q.enqueue("fetch", f"{n}".encode())
+    await q.enqueue(
+        ["fetch"] * N,
+        [f"{n}".encode() for n in range(N)],
+        [0] * N,
+    )
 
     # Stop flag
     await q.enqueue("fetch", None)
@@ -92,8 +98,11 @@ async def test_sync_entrypoint(
             assert context
             seen.append(int(context.payload))
 
-    for n in range(N):
-        await q.enqueue("fetch", f"{n}".encode())
+    await q.enqueue(
+        ["fetch"] * N,
+        [f"{n}".encode() for n in range(N)],
+        [0] * N,
+    )
 
     # Stop flag
     await q.enqueue("fetch", None)

{pgqueuer-0.2.2 → pgqueuer-0.2.4}/test/test_queries.py

@@ -24,8 +24,11 @@ async def test_queries_next_jobs(
 ) -> None:
     q = queries.Queries(pgpool)
 
-    for n in range(N):
-        await q.enqueue("placeholer", f"{n}".encode())
+    await q.enqueue(
+        ["placeholder"] * N,
+        [f"{n}".encode() for n in range(N)],
+        [0] * N,
+    )
 
     seen = list[int]()
     while job := await q.dequeue():
@@ -47,8 +50,11 @@ async def test_queries_next_jobs_concurrent(
     assert pgpool.get_max_size() >= concurrency
     q = queries.Queries(pgpool)
 
-    for n in range(N):
-        await q.enqueue("placeholer", f"{n}".encode())
+    await q.enqueue(
+        ["placeholder"] * N,
+        [f"{n}".encode() for n in range(N)],
+        [0] * N,
+    )
 
     seen = list[int]()
 
@@ -86,13 +92,16 @@ async def test_move_job_log(
 ) -> None:
     q = queries.Queries(pgpool)
 
-    for n in range(N):
-        await q.enqueue("placeholer", f"{n}".encode())
+    await q.enqueue(
+        ["placeholer"] * N,
+        [f"{n}".encode() for n in range(N)],
+        [0] * N,
+    )
 
-    while next_job := await q.dequeue():
-        await q.log_job(next_job, status="successful")
+    while job := await q.dequeue():
+        await q.log_job(job, status="successful")
 
-    assert sum(x.count for x in await q.log_size()) == N
+    assert sum(x.count for x in await q.log_statistics(1_000_000_000)) == N
 
 
 @pytest.mark.parametrize("N", (1, 2, 5))
@@ -103,8 +112,11 @@ async def test_clear_queue(
     q = queries.Queries(pgpool)
 
     # Test delete all by listing all
-    for n in range(N):
-        await q.enqueue(f"placeholer{n}", None)
+    await q.enqueue(
+        [f"placeholer{n}" for n in range(N)],
+        [None] * N,
+        [0] * N,
+    )
 
     assert all(x.count == 1 for x in await q.queue_size())
     assert sum(x.count for x in await q.queue_size()) == N
@@ -112,8 +124,11 @@ async def test_clear_queue(
     assert sum(x.count for x in await q.queue_size()) == 0
 
     # Test delete all by None
-    for n in range(N):
-        await q.enqueue(f"placeholer{n}", None)
+    await q.enqueue(
+        [f"placeholer{n}" for n in range(N)],
+        [None] * N,
+        [0] * N,
+    )
 
     assert all(x.count == 1 for x in await q.queue_size())
     assert sum(x.count for x in await q.queue_size()) == N
@@ -121,8 +136,11 @@ async def test_clear_queue(
     assert sum(x.count for x in await q.queue_size()) == 0
 
     # Test delete one(1).
-    for n in range(N):
-        await q.enqueue(f"placeholer{n}", None)
+    await q.enqueue(
+        [f"placeholer{n}" for n in range(N)],
+        [None] * N,
+        [0] * N,
+    )
 
     assert all(x.count == 1 for x in await q.queue_size())
     assert sum(x.count for x in await q.queue_size()) == N
@@ -138,8 +156,11 @@ async def test_queue_priority(
     q = queries.Queries(pgpool)
     jobs = list[models.Job]()
 
-    for n in range(N):
-        await q.enqueue("placeholer", f"{n}".encode(), priority=n)
+    await q.enqueue(
+        ["placeholder"] * N,
+        [f"{n}".encode() for n in range(N)],
+        list(range(N)),
+    )
 
     while next_job := await q.dequeue():
         jobs.append(next_job)

pgqueuer-0.2.4/tools/benchmark.py

@@ -0,0 +1,100 @@
+import argparse
+import asyncio
+import contextlib
+import random
+import time
+from datetime import datetime, timedelta
+from typing import Callable, Generator
+
+import asyncpg
+from PgQueuer.models import Job
+from PgQueuer.qm import QueueManager
+from PgQueuer.queries import Queries
+
+
+@contextlib.contextmanager
+def execution_timer() -> (
+    Generator[
+        Callable[[], timedelta],
+        None,
+        None,
+    ]
+):
+    enter = time.perf_counter()
+    done: float | None = None
+    try:
+        yield lambda: timedelta(seconds=((done or time.perf_counter()) - enter))
+    finally:
+        done = time.perf_counter()
+
+
+async def run_qm(pool: asyncpg.Pool) -> None:
+    qm = QueueManager(pool)
+
+    @qm.entrypoint("async")
+    async def asyncfetch(job: Job) -> None:
+        if job.payload is None:
+            qm.alive = False
+
+    @qm.entrypoint("sync")
+    def syncfetch(job: Job) -> None:
+        if job.payload is None:
+            qm.alive = False
+
+    await qm.run(timedelta(seconds=0))
+
+
+async def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Run the job processing benchmark.",
+    )
+    parser.add_argument(
+        "-t",
+        "--time-limit",
+        type=lambda x: timedelta(seconds=float(x)),
+        default=timedelta(seconds=10),
+        help="Run the benchmark for a limited number of seconds. Default is 10",
+    )
+    args = parser.parse_args()
+    start = datetime.now()
+
+    async with asyncpg.create_pool() as pool:
+        queries = Queries(pool)
+        await queries.clear_log()
+        while True:
+            for concurrecy in range(1, 6):
+                N = concurrecy * 1_000
+
+                await queries.clear_queue()
+
+                entrypoints = ["sync"] * (N // 2) + ["async"] * (N // 2)
+                random.shuffle(entrypoints)
+                payloads = [f"{n}".encode() for n in range(N)]
+                priorities = [0] * N
+                await queries.enqueue(entrypoints, payloads, priorities)
+                await queries.enqueue(
+                    ["async"] * concurrecy**2,
+                    [None] * concurrecy**2,
+                    [0] * concurrecy**2,
+                )
+
+                with execution_timer() as elapsed:
+                    await asyncio.gather(
+                        *[run_qm(pool) for _ in range(concurrecy)],
+                    )
+
+                elapsed_total_seconds = elapsed().total_seconds()
+                jps = (N - len(await queries.queue_size())) / elapsed_total_seconds
+
+                print(
+                    f"Concurrecy: {concurrecy:<2} ",
+                    f"Jobs: {N:<5} ",
+                    f"JPS: {(jps)/1_000:.1f}k",
+                )
+
+            if timedelta(seconds=0) < args.time_limit < datetime.now() - start:
+                return
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

pgqueuer-0.2.2/src/PgQueuer/__main__.py

@@ -1,6 +0,0 @@
-import asyncio
-
-from PgQueuer import cli
-
-if __name__ == "__main__":
-    asyncio.run(cli.main())

pgqueuer-0.2.2/tools/benchmark.py

@@ -1,99 +0,0 @@
-import asyncio
-import contextlib
-import time
-from datetime import timedelta
-from typing import Callable, Generator
-
-import asyncpg
-from PgQueuer.models import Job
-from PgQueuer.qm import QueueManager
-from PgQueuer.queries import Queries
-
-
-@contextlib.contextmanager
-def timer() -> (
-    Generator[
-        Callable[[], float],
-        None,
-        None,
-    ]
-):
-    enter = time.perf_counter()
-    done: float | None = None
-    try:
-        yield lambda: done - enter if done else time.perf_counter() - enter
-    finally:
-        done = time.perf_counter()
-
-
-async def jobs_per_second(pool: asyncpg.Pool) -> float:
-    qm = QueueManager(pool)
-    seen = 0
-
-    @qm.entrypoint("async")
-    async def asyncfetch(job: Job) -> None:
-        nonlocal qm, seen
-        seen += 1
-        if job.payload is None:
-            qm.alive = False
-
-    @qm.entrypoint("sync")
-    def syncfetch(job: Job) -> None:
-        nonlocal qm, seen
-        seen += 1
-        if job.payload is None:
-            qm.alive = False
-
-    with timer() as elapsed:
-        await qm.run(timedelta(seconds=0))
-
-    return seen / elapsed()
-
-
-async def benchmark(
-    entrypoint: str,
-    concurrecy: int,
-    N: int,
-    pool: asyncpg.Pool,
-) -> None:
-    queries = Queries(pool)
-
-    await queries.clear_log()
-    await queries.clear_queue()
-
-    await queries.enqueue(
-        [entrypoint] * N,
-        [f"{n}".encode() for n in range(N)],
-        [0] * N,
-    )
-    assert sum(x.count for x in await queries.queue_size()) == N
-
-    await queries.enqueue(
-        [entrypoint] * concurrecy**2,
-        [None] * concurrecy**2,
-        [0] * concurrecy**2,
-    )
-
-    jobs = [jobs_per_second(pool) for _ in range(concurrecy)]
-    results = await asyncio.gather(*jobs)
-
-    print(
-        f"Entrypoint: {entrypoint:<5} ",
-        f"Concurrecy: {concurrecy:<2} "
-        f"Jobs: {N:<5} "
-        f"JPS: {sum(results)/1_000:.1f}k",
-    )
-
-
-async def main() -> None:
-    async with asyncpg.create_pool(
-        min_size=20,
-        max_size=99,
-    ) as pool:
-        for entrypoint in ("sync", "async"):
-            for concurrecy in range(1, 6):
-                await benchmark(entrypoint, concurrecy, 1_000 * concurrecy, pool)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())