nsq2mariadb 0.1.2__tar.gz

nsq2mariadb-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Lars Wallenborn
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nsq2mariadb-0.1.2/PKG-INFO

@@ -0,0 +1,157 @@
+Metadata-Version: 2.4
+Name: nsq2mariadb
+Version: 0.1.2
+Summary: generic NSQ → MariaDB transporter with per-topic Python mapper classes
+Home-page: https://github.com/larsborn/nsq2mariadb
+Author: Lars Wallenborn
+Project-URL: Bug Tracker, https://github.com/larsborn/nsq2mariadb/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pynsq
+Requires-Dist: pymysql
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# nsq2mariadb
+
+Generic transporter for moving JSON messages from [NSQ](https://nsq.io/) topics into
+[MariaDB](https://mariadb.org/) tables. You write one Python `Mapper` class per
+topic; the framework handles the NSQ subscription, schema bootstrap, and
+transactional inserts.
+
+The shape mirrors [`nsq2arangodb`](https://github.com/larsborn/nsq2arangodb) —
+but because MariaDB is schema-full, the per-topic schema and JSON-to-row
+translation has to live in code rather than configuration.
+
+## Installation
+
+```bash
+pip install git+https://github.com/larsborn/nsq2mariadb.git@v0.1.0
+```
+
+Or for local development:
+
+```bash
+git clone https://github.com/larsborn/nsq2mariadb.git
+cd nsq2mariadb
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -U pip
+pip install -e .
+```
+
+Requires Python 3.9+. Dependencies: `pynsq`, `pymysql`.
+
+## Usage
+
+For each NSQ topic you want to consume, subclass `Mapper`:
+
+```python
+from nsq2mariadb import Mapper
+
+class OrderMapper(Mapper):
+    topic = "orders"
+    schema_sql = """
+        CREATE TABLE IF NOT EXISTS `order` (
+            id          INT PRIMARY KEY,
+            customer    VARCHAR(255) NOT NULL,
+            inserted_at DATETIME     NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS order_item (
+            order_id INT          NOT NULL,
+            position SMALLINT     NOT NULL,
+            sku      VARCHAR(32)  NOT NULL,
+            PRIMARY KEY (order_id, position),
+            FOREIGN KEY (order_id) REFERENCES `order`(id) ON DELETE CASCADE
+        );
+    """
+
+    def transform(self, doc):
+        yield "order", {
+            "id":          doc["id"],
+            "customer":    doc["customer"],
+            "inserted_at": doc["inserted_at"],
+        }
+        for i, sku in enumerate(doc.get("items", [])):
+            yield "order_item", {"order_id": doc["id"], "position": i, "sku": sku}
+```
+
+Then wire it into the runner:
+
+```python
+import logging
+from nsq2mariadb import MariaDBConfig, Nsq2MariaDB, NsqConfig
+
+logging.basicConfig(level=logging.INFO)
+
+runner = Nsq2MariaDB(
+    logger=logging.getLogger("nsq2mariadb"),
+    mariadb_config=MariaDBConfig(
+        host="mariadb", port=3306,
+        user="orders", password="secret", database="orders",
+    ),
+    nsq_config=NsqConfig(
+        address="nsq-nsqd-1", port=4150,
+        channel="nsq2mariadb",
+    ),
+    mappers=[OrderMapper()],
+)
+runner.run()
+```
+
+On startup the framework opens one pymysql connection, executes every mapper's
+`schema_sql` with `CREATE TABLE IF NOT EXISTS` semantics, and subscribes an
+`nsq.Reader` per mapper. Each NSQ message is decoded, fanned out through the
+mapper's `transform()`, and inserted in a single transaction with parameterized
+`INSERT IGNORE` statements (so re-published messages are idempotent as long as
+your primary key reflects content identity).
+
+## Error handling
+
+| Failure mode                       | Behavior                                                    |
+|------------------------------------|-------------------------------------------------------------|
+| JSON decode error                  | Logged with traceback, message FIN'd (dropped — won't fix). |
+| `pymysql.MySQLError` during insert | Transaction rolled back, message FIN'd, traceback logged.   |
+| Connection drop                    | pymysql raises, propagates, process exits — relies on container restart. |
+
+Schema mismatches (unknown column, missing table, FK violation) are programmer
+or schema bugs that loop forever if requeued, so we drop them loudly. Tune your
+log shipping accordingly.
+
+## Multiple topics per process
+
+Pass several mappers to one `Nsq2MariaDB` instance — pynsq supports multiple
+`Reader`s in one IOLoop. They share the database connection but each runs an
+independent NSQ subscription. Useful when a single project produces several
+related topics (e.g. `entries` + `runs`) that you want to land in the same DB.
+
+## Releasing
+
+Releases are auto-published to [PyPI](https://pypi.org/project/nsq2mariadb/)
+by `.github/workflows/publish.yml` on every `v*` tag, via PyPI's Trusted
+Publishers (OIDC — no API token stored in the repo).
+
+To cut a release:
+
+1. Bump the version in `setup.py`.
+2. Commit and push to `main`.
+3. Tag and push: `git tag v0.1.3 && git push origin v0.1.3`.
+
+The workflow builds an sdist + wheel and publishes them under the
+`pypi` GitHub environment.
+
+## License
+
+MIT.

nsq2mariadb-0.1.2/README.md

@@ -0,0 +1,131 @@
+# nsq2mariadb
+
+Generic transporter for moving JSON messages from [NSQ](https://nsq.io/) topics into
+[MariaDB](https://mariadb.org/) tables. You write one Python `Mapper` class per
+topic; the framework handles the NSQ subscription, schema bootstrap, and
+transactional inserts.
+
+The shape mirrors [`nsq2arangodb`](https://github.com/larsborn/nsq2arangodb) —
+but because MariaDB is schema-full, the per-topic schema and JSON-to-row
+translation has to live in code rather than configuration.
+
+## Installation
+
+```bash
+pip install git+https://github.com/larsborn/nsq2mariadb.git@v0.1.0
+```
+
+Or for local development:
+
+```bash
+git clone https://github.com/larsborn/nsq2mariadb.git
+cd nsq2mariadb
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -U pip
+pip install -e .
+```
+
+Requires Python 3.9+. Dependencies: `pynsq`, `pymysql`.
+
+## Usage
+
+For each NSQ topic you want to consume, subclass `Mapper`:
+
+```python
+from nsq2mariadb import Mapper
+
+class OrderMapper(Mapper):
+    topic = "orders"
+    schema_sql = """
+        CREATE TABLE IF NOT EXISTS `order` (
+            id          INT PRIMARY KEY,
+            customer    VARCHAR(255) NOT NULL,
+            inserted_at DATETIME     NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS order_item (
+            order_id INT          NOT NULL,
+            position SMALLINT     NOT NULL,
+            sku      VARCHAR(32)  NOT NULL,
+            PRIMARY KEY (order_id, position),
+            FOREIGN KEY (order_id) REFERENCES `order`(id) ON DELETE CASCADE
+        );
+    """
+
+    def transform(self, doc):
+        yield "order", {
+            "id":          doc["id"],
+            "customer":    doc["customer"],
+            "inserted_at": doc["inserted_at"],
+        }
+        for i, sku in enumerate(doc.get("items", [])):
+            yield "order_item", {"order_id": doc["id"], "position": i, "sku": sku}
+```
+
+Then wire it into the runner:
+
+```python
+import logging
+from nsq2mariadb import MariaDBConfig, Nsq2MariaDB, NsqConfig
+
+logging.basicConfig(level=logging.INFO)
+
+runner = Nsq2MariaDB(
+    logger=logging.getLogger("nsq2mariadb"),
+    mariadb_config=MariaDBConfig(
+        host="mariadb", port=3306,
+        user="orders", password="secret", database="orders",
+    ),
+    nsq_config=NsqConfig(
+        address="nsq-nsqd-1", port=4150,
+        channel="nsq2mariadb",
+    ),
+    mappers=[OrderMapper()],
+)
+runner.run()
+```
+
+On startup the framework opens one pymysql connection, executes every mapper's
+`schema_sql` with `CREATE TABLE IF NOT EXISTS` semantics, and subscribes an
+`nsq.Reader` per mapper. Each NSQ message is decoded, fanned out through the
+mapper's `transform()`, and inserted in a single transaction with parameterized
+`INSERT IGNORE` statements (so re-published messages are idempotent as long as
+your primary key reflects content identity).
+
+## Error handling
+
+| Failure mode                       | Behavior                                                    |
+|------------------------------------|-------------------------------------------------------------|
+| JSON decode error                  | Logged with traceback, message FIN'd (dropped — won't fix). |
+| `pymysql.MySQLError` during insert | Transaction rolled back, message FIN'd, traceback logged.   |
+| Connection drop                    | pymysql raises, propagates, process exits — relies on container restart. |
+
+Schema mismatches (unknown column, missing table, FK violation) are programmer
+or schema bugs that loop forever if requeued, so we drop them loudly. Tune your
+log shipping accordingly.
+
+## Multiple topics per process
+
+Pass several mappers to one `Nsq2MariaDB` instance — pynsq supports multiple
+`Reader`s in one IOLoop. They share the database connection but each runs an
+independent NSQ subscription. Useful when a single project produces several
+related topics (e.g. `entries` + `runs`) that you want to land in the same DB.
+
+## Releasing
+
+Releases are auto-published to [PyPI](https://pypi.org/project/nsq2mariadb/)
+by `.github/workflows/publish.yml` on every `v*` tag, via PyPI's Trusted
+Publishers (OIDC — no API token stored in the repo).
+
+To cut a release:
+
+1. Bump the version in `setup.py`.
+2. Commit and push to `main`.
+3. Tag and push: `git tag v0.1.3 && git push origin v0.1.3`.
+
+The workflow builds an sdist + wheel and publishes them under the
+`pypi` GitHub environment.
+
+## License
+
+MIT.

nsq2mariadb-0.1.2/nsq2mariadb/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from nsq2mariadb.nsq2mariadb import MariaDBConfig, Mapper, Nsq2MariaDB, NsqConfig

nsq2mariadb-0.1.2/nsq2mariadb/nsq2mariadb.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""Generic NSQ -> MariaDB transporter.
+
+Operators subscribe one or more `Mapper` subclasses (one per NSQ topic) to a
+running `Nsq2MariaDB` instance. Each mapper declares its target schema (DDL
+applied with CREATE TABLE IF NOT EXISTS on startup) and a `transform(doc)`
+method that yields `(table_name, row_dict)` tuples. The framework wraps every
+NSQ message in a single MariaDB transaction across all yielded rows and uses
+parameterized `INSERT IGNORE` for idempotency.
+"""
+import json
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Iterable, List, Sequence, Tuple
+
+import nsq
+import pymysql
+
+
+@dataclass
+class MariaDBConfig:
+    host: str
+    port: int
+    user: str
+    password: str
+    database: str
+    charset: str = "utf8mb4"
+
+
+@dataclass
+class NsqConfig:
+    address: str
+    port: int
+    channel: str
+    max_in_flight: int = 1
+
+
+class Mapper(ABC):
+    """Subclass per NSQ topic.
+
+    Set `topic` to the NSQ topic name and `schema_sql` to one or more
+    `CREATE TABLE IF NOT EXISTS` statements separated by semicolons.
+    Implement `transform()` to translate a decoded JSON message into rows.
+    """
+
+    topic: str = ""
+    schema_sql: str = ""
+
+    @abstractmethod
+    def transform(self, doc: dict) -> Iterable[Tuple[str, dict]]:
+        """Yield `(table_name, row_dict)` per row this message should insert."""
+
+
+def _split_statements(sql: str) -> List[str]:
+    """Split a multi-statement SQL string on `;` and drop empty fragments."""
+    return [stmt.strip() for stmt in sql.split(";") if stmt.strip()]
+
+
+def _build_insert(table: str, row: dict) -> Tuple[str, Sequence]:
+    """Build a parameterized `INSERT IGNORE` statement and its values tuple."""
+    if not row:
+        raise ValueError(f"refusing to insert empty row into {table!r}")
+    columns = list(row.keys())
+    col_list = ",".join(f"`{c}`" for c in columns)
+    placeholders = ",".join(["%s"] * len(columns))
+    sql = f"INSERT IGNORE INTO `{table}` ({col_list}) VALUES ({placeholders})"
+    return sql, tuple(row[c] for c in columns)
+
+
+class Nsq2MariaDB:
+    """Run one or more mappers against a single MariaDB connection."""
+
+    def __init__(
+        self,
+        logger: logging.Logger,
+        mariadb_config: MariaDBConfig,
+        nsq_config: NsqConfig,
+        mappers: Sequence[Mapper],
+        connection=None,
+    ):
+        if not mappers:
+            raise ValueError("at least one Mapper is required")
+        self._logger = logger
+        self._mariadb_config = mariadb_config
+        self._nsq_config = nsq_config
+        self._mappers = list(mappers)
+        self._conn = connection if connection is not None else self._open_connection()
+        self._apply_schemas()
+        self._register_readers()
+
+    def run(self) -> None:
+        """Enter the NSQ IOLoop. Returns when nsq.run() returns."""
+        nsq.run()
+
+    def _open_connection(self):
+        cfg = self._mariadb_config
+        return pymysql.connect(
+            host=cfg.host,
+            port=cfg.port,
+            user=cfg.user,
+            password=cfg.password,
+            database=cfg.database,
+            charset=cfg.charset,
+            autocommit=False,
+        )
+
+    def _apply_schemas(self) -> None:
+        for mapper in self._mappers:
+            statements = _split_statements(mapper.schema_sql)
+            if not statements:
+                continue
+            self._logger.info(
+                f"applying {len(statements)} schema statement(s) for topic {mapper.topic!r}"
+            )
+            with self._conn.cursor() as cur:
+                for stmt in statements:
+                    cur.execute(stmt)
+            self._conn.commit()
+
+    def _register_readers(self) -> None:
+        for mapper in self._mappers:
+            nsq.Reader(
+                message_handler=self._make_handler(mapper),
+                nsqd_tcp_addresses=[f"{self._nsq_config.address}:{self._nsq_config.port}"],
+                topic=mapper.topic,
+                channel=self._nsq_config.channel,
+                max_in_flight=self._nsq_config.max_in_flight,
+            )
+            self._logger.info(
+                f"subscribed to topic {mapper.topic!r} on channel {self._nsq_config.channel!r}"
+            )
+
+    def _make_handler(self, mapper: Mapper):
+        def handler(message) -> bool:
+            return self._handle_message(mapper, message)
+
+        return handler
+
+    def _handle_message(self, mapper: Mapper, message) -> bool:
+        try:
+            doc = json.loads(message.body.decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError):
+            self._logger.exception(
+                f"failed to decode message on topic {mapper.topic!r}; dropping"
+            )
+            return True  # FIN; broken JSON won't fix itself on retry
+        try:
+            with self._conn.cursor() as cur:
+                for table, row in mapper.transform(doc):
+                    sql, params = _build_insert(table, row)
+                    cur.execute(sql, params)
+            self._conn.commit()
+        except pymysql.MySQLError:
+            self._conn.rollback()
+            self._logger.exception(
+                f"database error handling message on topic {mapper.topic!r}; dropping"
+            )
+            return True  # FIN; matches nsq2arangodb behavior — programmer/data bugs
+            #                won't fix themselves on retry, so don't loop forever
+        return True

nsq2mariadb-0.1.2/nsq2mariadb.egg-info/PKG-INFO

@@ -0,0 +1,157 @@
+Metadata-Version: 2.4
+Name: nsq2mariadb
+Version: 0.1.2
+Summary: generic NSQ → MariaDB transporter with per-topic Python mapper classes
+Home-page: https://github.com/larsborn/nsq2mariadb
+Author: Lars Wallenborn
+Project-URL: Bug Tracker, https://github.com/larsborn/nsq2mariadb/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pynsq
+Requires-Dist: pymysql
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# nsq2mariadb
+
+Generic transporter for moving JSON messages from [NSQ](https://nsq.io/) topics into
+[MariaDB](https://mariadb.org/) tables. You write one Python `Mapper` class per
+topic; the framework handles the NSQ subscription, schema bootstrap, and
+transactional inserts.
+
+The shape mirrors [`nsq2arangodb`](https://github.com/larsborn/nsq2arangodb) —
+but because MariaDB is schema-full, the per-topic schema and JSON-to-row
+translation has to live in code rather than configuration.
+
+## Installation
+
+```bash
+pip install git+https://github.com/larsborn/nsq2mariadb.git@v0.1.0
+```
+
+Or for local development:
+
+```bash
+git clone https://github.com/larsborn/nsq2mariadb.git
+cd nsq2mariadb
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -U pip
+pip install -e .
+```
+
+Requires Python 3.9+. Dependencies: `pynsq`, `pymysql`.
+
+## Usage
+
+For each NSQ topic you want to consume, subclass `Mapper`:
+
+```python
+from nsq2mariadb import Mapper
+
+class OrderMapper(Mapper):
+    topic = "orders"
+    schema_sql = """
+        CREATE TABLE IF NOT EXISTS `order` (
+            id          INT PRIMARY KEY,
+            customer    VARCHAR(255) NOT NULL,
+            inserted_at DATETIME     NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS order_item (
+            order_id INT          NOT NULL,
+            position SMALLINT     NOT NULL,
+            sku      VARCHAR(32)  NOT NULL,
+            PRIMARY KEY (order_id, position),
+            FOREIGN KEY (order_id) REFERENCES `order`(id) ON DELETE CASCADE
+        );
+    """
+
+    def transform(self, doc):
+        yield "order", {
+            "id":          doc["id"],
+            "customer":    doc["customer"],
+            "inserted_at": doc["inserted_at"],
+        }
+        for i, sku in enumerate(doc.get("items", [])):
+            yield "order_item", {"order_id": doc["id"], "position": i, "sku": sku}
+```
+
+Then wire it into the runner:
+
+```python
+import logging
+from nsq2mariadb import MariaDBConfig, Nsq2MariaDB, NsqConfig
+
+logging.basicConfig(level=logging.INFO)
+
+runner = Nsq2MariaDB(
+    logger=logging.getLogger("nsq2mariadb"),
+    mariadb_config=MariaDBConfig(
+        host="mariadb", port=3306,
+        user="orders", password="secret", database="orders",
+    ),
+    nsq_config=NsqConfig(
+        address="nsq-nsqd-1", port=4150,
+        channel="nsq2mariadb",
+    ),
+    mappers=[OrderMapper()],
+)
+runner.run()
+```
+
+On startup the framework opens one pymysql connection, executes every mapper's
+`schema_sql` with `CREATE TABLE IF NOT EXISTS` semantics, and subscribes an
+`nsq.Reader` per mapper. Each NSQ message is decoded, fanned out through the
+mapper's `transform()`, and inserted in a single transaction with parameterized
+`INSERT IGNORE` statements (so re-published messages are idempotent as long as
+your primary key reflects content identity).
+
+## Error handling
+
+| Failure mode                       | Behavior                                                    |
+|------------------------------------|-------------------------------------------------------------|
+| JSON decode error                  | Logged with traceback, message FIN'd (dropped — won't fix). |
+| `pymysql.MySQLError` during insert | Transaction rolled back, message FIN'd, traceback logged.   |
+| Connection drop                    | pymysql raises, propagates, process exits — relies on container restart. |
+
+Schema mismatches (unknown column, missing table, FK violation) are programmer
+or schema bugs that loop forever if requeued, so we drop them loudly. Tune your
+log shipping accordingly.
+
+## Multiple topics per process
+
+Pass several mappers to one `Nsq2MariaDB` instance — pynsq supports multiple
+`Reader`s in one IOLoop. They share the database connection but each runs an
+independent NSQ subscription. Useful when a single project produces several
+related topics (e.g. `entries` + `runs`) that you want to land in the same DB.
+
+## Releasing
+
+Releases are auto-published to [PyPI](https://pypi.org/project/nsq2mariadb/)
+by `.github/workflows/publish.yml` on every `v*` tag, via PyPI's Trusted
+Publishers (OIDC — no API token stored in the repo).
+
+To cut a release:
+
+1. Bump the version in `setup.py`.
+2. Commit and push to `main`.
+3. Tag and push: `git tag v0.1.3 && git push origin v0.1.3`.
+
+The workflow builds an sdist + wheel and publishes them under the
+`pypi` GitHub environment.
+
+## License
+
+MIT.

nsq2mariadb-0.1.2/nsq2mariadb.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+LICENSE
+README.md
+pyproject.toml
+setup.cfg
+setup.py
+nsq2mariadb/__init__.py
+nsq2mariadb/nsq2mariadb.py
+nsq2mariadb.egg-info/PKG-INFO
+nsq2mariadb.egg-info/SOURCES.txt
+nsq2mariadb.egg-info/dependency_links.txt
+nsq2mariadb.egg-info/requires.txt
+nsq2mariadb.egg-info/top_level.txt
+tests/test_nsq2mariadb.py

nsq2mariadb-0.1.2/nsq2mariadb.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

nsq2mariadb-0.1.2/nsq2mariadb.egg-info/requires.txt

@@ -0,0 +1,2 @@
+pynsq
+pymysql

nsq2mariadb-0.1.2/nsq2mariadb.egg-info/top_level.txt

@@ -0,0 +1 @@
+nsq2mariadb

nsq2mariadb-0.1.2/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"

nsq2mariadb-0.1.2/setup.cfg

@@ -0,0 +1,7 @@
+[metadata]
+license_files = LICENSE
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

nsq2mariadb-0.1.2/setup.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import setuptools
+
+with open("README.md", "r", encoding="utf-8") as fp:
+    long_description = fp.read()
+
+setuptools.setup(
+    name="nsq2mariadb",
+    version="0.1.2",
+    author="Lars Wallenborn",
+    description="generic NSQ → MariaDB transporter with per-topic Python mapper classes",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/larsborn/nsq2mariadb",
+    project_urls={
+        "Bug Tracker": "https://github.com/larsborn/nsq2mariadb/issues",
+    },
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    install_requires=[
+        "pynsq",
+        "pymysql",
+    ],
+    packages=setuptools.find_packages(exclude=["tests", "tests.*"]),
+    python_requires=">=3.9",
+)

nsq2mariadb-0.1.2/tests/test_nsq2mariadb.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import json
+import logging
+import unittest
+from unittest.mock import MagicMock
+
+import pymysql
+
+from nsq2mariadb.nsq2mariadb import (
+    MariaDBConfig,
+    Mapper,
+    Nsq2MariaDB,
+    NsqConfig,
+    _build_insert,
+    _split_statements,
+)
+
+
+def _make_logger():
+    return logging.getLogger("test_nsq2mariadb")
+
+
+def _silent_init(runner: Nsq2MariaDB) -> None:
+    """Stand-in for `_register_readers` — we don't want to touch real NSQ in tests."""
+    return None
+
+
+class _StubCursor:
+    """Records every `execute(sql, params)` call and behaves as a context manager."""
+
+    def __init__(self):
+        self.calls = []  # list of (sql, params) tuples
+        self.raise_on_execute = None  # set to an Exception instance to raise
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        return False
+
+    def execute(self, sql, params=None):
+        self.calls.append((sql, params))
+        if self.raise_on_execute is not None:
+            raise self.raise_on_execute
+
+
+class _StubConnection:
+    def __init__(self):
+        self.cursor_obj = _StubCursor()
+        self.commits = 0
+        self.rollbacks = 0
+
+    def cursor(self):
+        return self.cursor_obj
+
+    def commit(self):
+        self.commits += 1
+
+    def rollback(self):
+        self.rollbacks += 1
+
+
+class _FakeMessage:
+    def __init__(self, payload: dict):
+        self.body = json.dumps(payload).encode("utf-8")
+
+
+class _SingleTableMapper(Mapper):
+    topic = "things"
+    schema_sql = "CREATE TABLE IF NOT EXISTS thing (`id` INT PRIMARY KEY, name VARCHAR(64))"
+
+    def transform(self, doc):
+        yield "thing", {"id": doc["id"], "name": doc["name"]}
+
+
+class _MultiTableMapper(Mapper):
+    topic = "orders"
+    schema_sql = (
+        "CREATE TABLE IF NOT EXISTS `order` (`id` INT PRIMARY KEY);\n"
+        "CREATE TABLE IF NOT EXISTS order_item (order_id INT, position INT, sku VARCHAR(32));"
+    )
+
+    def transform(self, doc):
+        yield "order", {"id": doc["id"]}
+        for i, sku in enumerate(doc.get("items", [])):
+            yield "order_item", {"order_id": doc["id"], "position": i, "sku": sku}
+
+
+def _build_runner(mapper, connection):
+    runner = Nsq2MariaDB.__new__(Nsq2MariaDB)
+    runner._logger = _make_logger()
+    runner._mariadb_config = MariaDBConfig(
+        host="localhost", port=3306, user="u", password="p", database="d"
+    )
+    runner._nsq_config = NsqConfig(address="127.0.0.1", port=4150, channel="test")
+    runner._mappers = [mapper]
+    runner._conn = connection
+    return runner
+
+
+class TestSplitStatements(unittest.TestCase):
+    def test_splits_on_semicolons_and_drops_blanks(self):
+        sql = "CREATE TABLE a (x INT);   ;\nCREATE TABLE b (y INT);\n\n"
+        self.assertEqual(
+            _split_statements(sql),
+            ["CREATE TABLE a (x INT)", "CREATE TABLE b (y INT)"],
+        )
+
+    def test_empty_string(self):
+        self.assertEqual(_split_statements(""), [])
+
+    def test_single_statement_no_trailing_semicolon(self):
+        self.assertEqual(_split_statements("SELECT 1"), ["SELECT 1"])
+
+
+class TestBuildInsert(unittest.TestCase):
+    def test_single_column(self):
+        sql, params = _build_insert("t", {"a": 1})
+        self.assertEqual(sql, "INSERT IGNORE INTO `t` (`a`) VALUES (%s)")
+        self.assertEqual(params, (1,))
+
+    def test_multiple_columns_preserve_dict_order(self):
+        sql, params = _build_insert("zvg_entry", {"_key": "abc", "land_short": "by", "zvg_id": 7})
+        self.assertEqual(
+            sql,
+            "INSERT IGNORE INTO `zvg_entry` (`_key`,`land_short`,`zvg_id`) VALUES (%s,%s,%s)",
+        )
+        self.assertEqual(params, ("abc", "by", 7))
+
+    def test_empty_row_raises(self):
+        with self.assertRaises(ValueError):
+            _build_insert("t", {})
+
+
+class TestHandleMessage(unittest.TestCase):
+    def test_single_table_inserts_and_commits(self):
+        conn = _StubConnection()
+        runner = _build_runner(_SingleTableMapper(), conn)
+
+        result = runner._handle_message(runner._mappers[0], _FakeMessage({"id": 42, "name": "x"}))
+
+        self.assertTrue(result)
+        self.assertEqual(conn.commits, 1)
+        self.assertEqual(conn.rollbacks, 0)
+        self.assertEqual(len(conn.cursor_obj.calls), 1)
+        sql, params = conn.cursor_obj.calls[0]
+        self.assertEqual(sql, "INSERT IGNORE INTO `thing` (`id`,`name`) VALUES (%s,%s)")
+        self.assertEqual(params, (42, "x"))
+
+    def test_multi_table_fanout_is_one_transaction(self):
+        conn = _StubConnection()
+        runner = _build_runner(_MultiTableMapper(), conn)
+
+        result = runner._handle_message(
+            runner._mappers[0],
+            _FakeMessage({"id": 7, "items": ["sku-A", "sku-B", "sku-C"]}),
+        )
+
+        self.assertTrue(result)
+        self.assertEqual(conn.commits, 1)  # exactly one commit for the whole fanout
+        self.assertEqual(conn.rollbacks, 0)
+        self.assertEqual(len(conn.cursor_obj.calls), 4)  # 1 order + 3 items
+        # First call is the parent row
+        sql, params = conn.cursor_obj.calls[0]
+        self.assertEqual(sql, "INSERT IGNORE INTO `order` (`id`) VALUES (%s)")
+        self.assertEqual(params, (7,))
+        # Subsequent calls are the items with preserved order
+        for i, sku in enumerate(["sku-A", "sku-B", "sku-C"]):
+            sql, params = conn.cursor_obj.calls[i + 1]
+            self.assertEqual(
+                sql,
+                "INSERT IGNORE INTO `order_item` (`order_id`,`position`,`sku`) VALUES (%s,%s,%s)",
+            )
+            self.assertEqual(params, (7, i, sku))
+
+    def test_db_error_rolls_back_and_drops_message(self):
+        conn = _StubConnection()
+        conn.cursor_obj.raise_on_execute = pymysql.MySQLError("table missing")
+        runner = _build_runner(_SingleTableMapper(), conn)
+
+        result = runner._handle_message(runner._mappers[0], _FakeMessage({"id": 1, "name": "x"}))
+
+        self.assertTrue(result)  # FIN — do not requeue
+        self.assertEqual(conn.commits, 0)
+        self.assertEqual(conn.rollbacks, 1)
+
+    def test_json_decode_error_drops_message(self):
+        conn = _StubConnection()
+        runner = _build_runner(_SingleTableMapper(), conn)
+
+        bad_message = MagicMock()
+        bad_message.body = b"\xff\xfenot json"
+
+        result = runner._handle_message(runner._mappers[0], bad_message)
+
+        self.assertTrue(result)
+        self.assertEqual(conn.commits, 0)
+        self.assertEqual(conn.rollbacks, 0)
+        self.assertEqual(len(conn.cursor_obj.calls), 0)
+
+
+class TestApplySchemas(unittest.TestCase):
+    def test_executes_each_statement_and_commits(self):
+        conn = _StubConnection()
+        runner = _build_runner(_MultiTableMapper(), conn)
+
+        # Re-apply manually (the constructor isn't called in our test fixture)
+        runner._apply_schemas()
+
+        # 2 statements in _MultiTableMapper.schema_sql
+        self.assertEqual(len(conn.cursor_obj.calls), 2)
+        self.assertIn("CREATE TABLE IF NOT EXISTS `order`", conn.cursor_obj.calls[0][0])
+        self.assertIn("CREATE TABLE IF NOT EXISTS order_item", conn.cursor_obj.calls[1][0])
+        self.assertEqual(conn.commits, 1)
+
+
+if __name__ == "__main__":
+    unittest.main()