PgQueuer 0.1.0__tar.gz

pgqueuer-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,38 @@
+name: Test PgQueuer
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+jobs:
+  ci:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        os: [ubuntu-latest]
+
+    name: PY ${{ matrix.python-version }} on ${{ matrix.os }} 
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build custom PostgreSQL Docker image
+        run: |
+          docker-compose up --build -d pgq
+
+      - name: Install PgQueuer
+        run: |
+          pip install pip -U
+          pip install .[dev]
+
+      - name: Full test
+        run: pytest -v

pgqueuer-0.1.0/.github/workflows/linting.yml

@@ -0,0 +1,46 @@
+name: linting
+
+on: [push, pull_request]
+
+jobs:
+
+  ruff:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up python
+        uses: actions/setup-python@v4
+      - name: Install dev-env.
+        run: |
+          pip install -U pip
+          pip install ".[dev]"
+      - name: Ruff check
+        if: ${{ always() }}
+        run: ruff check .
+      - name: Ruff format
+        if: ${{ always() }}
+        run: ruff format . --check
+
+  mypy:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dev-env.
+        run: |
+          pip install -U pip
+          pip install ".[dev]"
+
+      - name: Mypy
+        run: mypy .

pgqueuer-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,35 @@
+name: Publish Python distribution to PyPI
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-and-publish:
+    name: Publish Python distribution to PyPI.
+    runs-on: ubuntu-latest
+
+    environment:
+      name: release
+      url: https://pypi.org/project/PgQueuer/
+
+    permissions:
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Ensures tags are also fetched
+
+      - name: Set up Python 3.10.
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install release dependencies.
+        run: python3 -m pip install build twine setuptools_scm
+
+      - name: Build package.
+        run: python3 -m build . --sdist --wheel --outdir dist/
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

pgqueuer-0.1.0/.gitignore

@@ -0,0 +1,162 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+src/PgQueuer/_version.py
+src/_version.py

pgqueuer-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 janbjorge
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pgqueuer-0.1.0/PKG-INFO

@@ -0,0 +1,140 @@
+Metadata-Version: 2.1
+Name: PgQueuer
+Version: 0.1.0
+Summary: PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing.
+Author: janbjorge
+License: MIT License
+Project-URL: Documentation, https://github.com/janbjorge/PgQueuer/
+Project-URL: Homepage, https://github.com/janbjorge/PgQueuer/
+Project-URL: Issues, https://github.com/janbjorge/PgQueuer/issues
+Project-URL: Repository, https://github.com/janbjorge/PgQueuer/
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Web Environment
+Classifier: Framework :: AsyncIO
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Utilities
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: asyncpg>=0.27.0
+Requires-Dist: pgcachewatch>=0.4
+Requires-Dist: pydantic>=2.0.0
+Provides-Extra: dev
+Requires-Dist: asyncpg-stubs; extra == "dev"
+Requires-Dist: mypy-extensions; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: pytest-asyncio; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: myst-parser; extra == "docs"
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-rtd-theme; extra == "docs"
+
+## PgQueuer
+
+PgQueuer is a minimalist, high-performance job queue library for Python, leveraging the robustness of PostgreSQL. Designed for simplicity and efficiency, PgQueuer uses PostgreSQL's native features like transactions, row locking, and LISTEN/NOTIFY to manage job queues effortlessly.
+
+### Features
+
+- **Simple Integration**: Easy to integrate with existing Python applications using PostgreSQL.
+- **Efficient Concurrency Handling**: Utilizes PostgreSQL's `FOR UPDATE SKIP LOCKED` for reliable and concurrent job processing.
+- **Real-time Notifications**: Leverages `LISTEN` and `NOTIFY` for real-time updates on job status changes.
+- **Scalable**: Built to efficiently scale with your application's needs.
+
+### Getting Started
+
+#### Installation
+
+To install PgQueuer, simply run the following command:
+
+```bash
+pip install PgQueuer
+```
+
+This command installs PgQueuer along with its dependencies, setting up everything you need to start integrating it into your application.
+
+#### Database Configuration
+
+Configure your database connection settings by setting the appropriate environment variables for your PostgreSQL credentials:
+
+```bash
+export PGHOST='localhost'
+export PGDATABASE='your_database'
+export PGUSER='your_username'
+export PGPASSWORD='your_password'
+```
+
+#### CLI Usage
+
+PgQueuer provides a command-line interface for easy management of installation and uninstallation. Ensure you have configured your environment variables or use the appropriate flags to specify your database credentials.
+
+- **Installing PgQueuer Database Components**:
+  ```bash
+  python -m PgQueuer install 
+  ```
+
+- **Uninstalling PgQueuer Database Components**:
+  ```bash
+  python -m  uninstall 
+  ```
+
+The CLI supports several flags to customize the connection settings. Use `--help` to see all available options.
+
+### Use-Case: Processing Incoming Data Messages
+
+In this scenario, the system is designed to manage a queue of incoming data messages that need to be processed. Each message is encapsulated as a job, and these jobs are then processed by a designated function linked to an entrypoint. This is typical in systems where large volumes of data are collected continuously, such as telemetry data from IoT devices, logs from web servers, or transaction data in financial systems.
+
+#### Function Description:
+The `fetch` function, tied to the `fetch` entrypoint, is responsible for processing each message. It simulates a processing task by printing the content of each message. This function could be adapted to perform more complex operations such as parsing, analyzing, storing, or forwarding the data as required by the application.
+
+```python
+import asyncio
+import time
+
+import asyncpg
+
+from PgQueuer import qm, queries
+
+
+async def main() -> None:
+    # Set up a connection pool with a minimum of 2 connections.
+    pool = await asyncpg.create_pool(min_size=2)
+    # Initialize the QueueManager with the connection pool and query handler.
+    app = qm.QueueManager(
+        pool, 
+        queries.PgQueuerQueries(pool),
+        queries.PgQueuerLogQueries(pool),
+    )
+
+    # Number of messages to simulate.
+    N = 10_000
+
+    # Enqueue messages.
+    for n in range(N):
+        await app.q.enqueue("fetch", f"this is from me: {n}".encode())
+
+    # Define a processing function for each message.
+    @c.entrypoint("fetch")
+    async def process_message(context: bytes) -> None:
+        # Simulate parsing the message.
+        message = context.decode()
+        # Print the message to simulate processing.
+        print(f"Processed message: {message}")
+
+    await app.run()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```

pgqueuer-0.1.0/README.md

@@ -0,0 +1,97 @@
+## PgQueuer
+
+PgQueuer is a minimalist, high-performance job queue library for Python, leveraging the robustness of PostgreSQL. Designed for simplicity and efficiency, PgQueuer uses PostgreSQL's native features like transactions, row locking, and LISTEN/NOTIFY to manage job queues effortlessly.
+
+### Features
+
+- **Simple Integration**: Easy to integrate with existing Python applications using PostgreSQL.
+- **Efficient Concurrency Handling**: Utilizes PostgreSQL's `FOR UPDATE SKIP LOCKED` for reliable and concurrent job processing.
+- **Real-time Notifications**: Leverages `LISTEN` and `NOTIFY` for real-time updates on job status changes.
+- **Scalable**: Built to efficiently scale with your application's needs.
+
+### Getting Started
+
+#### Installation
+
+To install PgQueuer, simply run the following command:
+
+```bash
+pip install PgQueuer
+```
+
+This command installs PgQueuer along with its dependencies, setting up everything you need to start integrating it into your application.
+
+#### Database Configuration
+
+Configure your database connection settings by setting the appropriate environment variables for your PostgreSQL credentials:
+
+```bash
+export PGHOST='localhost'
+export PGDATABASE='your_database'
+export PGUSER='your_username'
+export PGPASSWORD='your_password'
+```
+
+#### CLI Usage
+
+PgQueuer provides a command-line interface for easy management of installation and uninstallation. Ensure you have configured your environment variables or use the appropriate flags to specify your database credentials.
+
+- **Installing PgQueuer Database Components**:
+  ```bash
+  python -m PgQueuer install 
+  ```
+
+- **Uninstalling PgQueuer Database Components**:
+  ```bash
+  python -m  uninstall 
+  ```
+
+The CLI supports several flags to customize the connection settings. Use `--help` to see all available options.
+
+### Use-Case: Processing Incoming Data Messages
+
+In this scenario, the system is designed to manage a queue of incoming data messages that need to be processed. Each message is encapsulated as a job, and these jobs are then processed by a designated function linked to an entrypoint. This is typical in systems where large volumes of data are collected continuously, such as telemetry data from IoT devices, logs from web servers, or transaction data in financial systems.
+
+#### Function Description:
+The `fetch` function, tied to the `fetch` entrypoint, is responsible for processing each message. It simulates a processing task by printing the content of each message. This function could be adapted to perform more complex operations such as parsing, analyzing, storing, or forwarding the data as required by the application.
+
+```python
+import asyncio
+import time
+
+import asyncpg
+
+from PgQueuer import qm, queries
+
+
+async def main() -> None:
+    # Set up a connection pool with a minimum of 2 connections.
+    pool = await asyncpg.create_pool(min_size=2)
+    # Initialize the QueueManager with the connection pool and query handler.
+    app = qm.QueueManager(
+        pool, 
+        queries.PgQueuerQueries(pool),
+        queries.PgQueuerLogQueries(pool),
+    )
+
+    # Number of messages to simulate.
+    N = 10_000
+
+    # Enqueue messages.
+    for n in range(N):
+        await app.q.enqueue("fetch", f"this is from me: {n}".encode())
+
+    # Define a processing function for each message.
+    @c.entrypoint("fetch")
+    async def process_message(context: bytes) -> None:
+        # Simulate parsing the message.
+        message = context.decode()
+        # Print the message to simulate processing.
+        print(f"Processed message: {message}")
+
+    await app.run()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```

pgqueuer-0.1.0/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3.10'
+services:
+  pgq:
+    build:
+      context: .
+      dockerfile: test/db/Dockerfile
+    ports:
+      - "5432:5432"
+    tmpfs:
+      - /var/lib/pg/data
+    environment:
+      PGDATA: /var/lib/pg/data
+    restart: "no"

pgqueuer-0.1.0/pyproject.toml

@@ -0,0 +1,94 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = ["pip>=22.0.0", "setuptools", "setuptools_scm", "wheel"]
+
+[project]
+authors = [{ name = "janbjorge"}]
+description = "PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing."
+dynamic = ["version"]
+license = { text = "MIT License" }
+name = "PgQueuer"
+readme = "README.md"
+requires-python = ">=3.10"
+
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Web Environment",
+    "Framework :: AsyncIO",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python",
+    "Topic :: Database",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Utilities",
+    "Topic :: System :: Distributed Computing"
+]
+
+dependencies = [
+    "asyncpg>=0.27.0",
+    "pgcachewatch>=0.4",
+    "pydantic>=2.0.0",
+]
+
+[project.urls]
+Documentation = "https://github.com/janbjorge/PgQueuer/"
+Homepage = "https://github.com/janbjorge/PgQueuer/"
+Issues = "https://github.com/janbjorge/PgQueuer/issues"
+Repository = "https://github.com/janbjorge/PgQueuer/"
+
+[project.optional-dependencies]
+dev = [
+    "asyncpg-stubs",
+    "mypy-extensions",
+    "mypy",
+    "pytest-asyncio",
+    "pytest",
+    "ruff",
+]
+docs = [
+    "myst-parser",
+    "sphinx",
+    "sphinx-rtd-theme",
+]
+
+[tool.setuptools_scm]
+write_to = "src/_version.py"
+
+[tool.ruff]
+line-length = 88
+[tool.ruff.lint]
+select = [
+    "C",
+    "E",
+    "F",
+    "I",
+    "PIE",
+    "Q",
+    "RET",
+    "RSE",
+    "SIM",
+    "W",
+    "C90",
+]
+[tool.ruff.lint.isort]
+combine-as-imports = true
+
+[tool.mypy]
+disallow_untyped_defs = true
+exclude = "^(build|docs)"
+extra_checks = true
+ignore_missing_imports = true
+plugins = ["pydantic.mypy"]
+python_version = "3.10"
+strict_equality = true
+warn_redundant_casts = true
+warn_unused_configs = true
+warn_unused_ignores = true
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

pgqueuer-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pgqueuer-0.1.0/src/PgQueuer/__main__.py

@@ -0,0 +1,6 @@
+import asyncio
+
+from PgQueuer import cli
+
+if __name__ == "__main__":
+    asyncio.run(cli.main())

pgqueuer-0.1.0/src/PgQueuer/_version.py

@@ -0,0 +1,17 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = "0.1.dev1+gac06244.d20240420"
+__version_tuple__ = version_tuple = (0, 1, "dev1", "gac06244.d20240420")