stream2pg 0.1.0__py3-none-any.whl

stream2pg/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from stream2pg.sink import Stream2Pg
+
+__all__ = ["Stream2Pg"]

stream2pg/config.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+from enum import Enum
+from typing import Any, Callable, Optional
+
+from .errors import ConfigurationError
+
+
+class ErrorStrategy(Enum):
+    RAISE = "raise"
+    SKIP_ON_ERROR = "skip_on_error"
+
+
+def from_config(
+    config: dict[str, Any],
+    on_metrics: Optional[Callable[..., None]] = None,
+) -> dict[str, Any]:
+    missing = []
+    for key in ("postgres", "kafka", "processing"):
+        if key not in config:
+            missing.append(key)
+
+    if missing:
+        raise ConfigurationError(f"Missing required config keys: {', '.join(missing)}")
+
+    return {
+        "postgres": config["postgres"],
+        "kafka": config["kafka"],
+        "processing": config["processing"],
+        "on_metrics": on_metrics,
+    }

stream2pg/core.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+import json
+import time
+from typing import Any, Callable, Optional
+
+from pyspark.sql import SparkSession
+from pyspark.sql.functions import col
+from pyspark.sql.types import StructType
+
+from .config import ErrorStrategy
+from .db import ensure_columns, ensure_table, insert_record
+
+
+def create_spark_session(app_name: str = "KafkaToPostgres") -> SparkSession:
+    return (
+        SparkSession.builder.appName(app_name)
+        .config(
+            "spark.jars.packages",
+            "org.apache.spark:spark-sql-kafka-0-10_2.13:4.1.2",
+        )
+        .getOrCreate()
+    )
+
+
+def process_batch(
+    batch_df: Any,
+    batch_id: int,
+    pg_config: dict[str, Any],
+    error_strategy: ErrorStrategy,
+    on_metrics: Optional[Callable[..., None]] = None,
+) -> None:
+    start_time = time.time()
+    row_count = 0
+    inserted_count = 0
+    skipped_count = 0
+
+    if batch_df.isEmpty():
+        return
+
+    row_count = batch_df.count()
+    conn = pg_config.get("connection")
+    if conn is None:
+        import psycopg2
+
+        conn = psycopg2.connect(
+            **{k: v for k, v in pg_config.items() if k != "connection"}
+        )
+        owns_connection = True
+    else:
+        owns_connection = False
+
+    try:
+        known_columns: dict[str, frozenset] = {}
+        for row in batch_df.toLocalIterator():
+            table_name = row.topic.replace("mobility-", "")
+            try:
+                record = json.loads(row.value)
+            except json.JSONDecodeError:
+                skipped_count += 1
+                if error_strategy == ErrorStrategy.RAISE:
+                    raise
+                continue
+
+            ensure_table(conn, table_name)
+            current_schema = frozenset(record.keys())
+            if known_columns.get(table_name) != current_schema:
+                ensure_columns(conn, table_name, record)
+                known_columns[table_name] = current_schema
+
+            try:
+                insert_record(conn, table_name, record)
+                inserted_count += 1
+            except Exception:
+                if error_strategy == ErrorStrategy.RAISE:
+                    raise
+                skipped_count += 1
+
+        conn.commit()
+    except Exception:
+        conn.rollback()
+        raise
+    finally:
+        if owns_connection:
+            conn.close()
+
+    elapsed_ms = int((time.time() - start_time) * 1000)
+    if on_metrics:
+        on_metrics(batch_id, row_count, inserted_count, skipped_count, elapsed_ms)
+
+
+def create_kafka_stream(
+    spark: SparkSession,
+    kafka_config: dict[str, Any],
+    schema: Optional[StructType] = None,
+):
+    reader = (
+        spark.readStream.format("kafka")
+        .option(
+            "kafka.bootstrap.servers",
+            kafka_config.get("bootstrap_servers", "localhost:9092"),
+        )
+        .option(
+            "subscribePattern", kafka_config.get("subscribe_pattern", "mobility-.*")
+        )
+        .option("startingOffsets", kafka_config.get("starting_offsets", "earliest"))
+        .option(
+            "failOnDataLoss",
+            str(kafka_config.get("fail_on_data_loss", "false")).lower(),
+        )
+    )
+
+    df = reader.load()
+    df = df.select(
+        col("topic"),
+        col("value").cast("string").alias("value"),
+    )
+    return df

stream2pg/db.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+from datetime import datetime
+from typing import Any
+
+import psycopg2
+from psycopg2 import sql
+
+
+def infer_sql_type(value: Any) -> str:
+    if isinstance(value, bool):
+        return "BOOLEAN"
+    if isinstance(value, int):
+        return "BIGINT"
+    if isinstance(value, float):
+        return "DOUBLE PRECISION"
+    if isinstance(value, str):
+        try:
+            datetime.fromisoformat(value.replace("Z", "+00:00"))
+            return "TIMESTAMP"
+        except Exception:
+            return "TEXT"
+    return "TEXT"
+
+
+def ensure_table(conn: psycopg2.extensions.connection, table_name: str) -> None:
+    with conn.cursor() as cur:
+        cur.execute(
+            sql.SQL("CREATE TABLE IF NOT EXISTS {} (id BIGSERIAL PRIMARY KEY)").format(
+                sql.Identifier(table_name)
+            )
+        )
+
+
+def ensure_columns(
+    conn: psycopg2.extensions.connection, table_name: str, record: dict
+) -> None:
+    with conn.cursor() as cur:
+        for key, value in record.items():
+            cur.execute(
+                """
+                SELECT EXISTS (
+                    SELECT 1
+                    FROM information_schema.columns
+                    WHERE table_name = %s
+                    AND column_name = %s
+                )
+                """,
+                (table_name, key),
+            )
+            exists = cur.fetchone()[0]
+            if not exists:
+                cur.execute(
+                    sql.SQL("ALTER TABLE {} ADD COLUMN {} {}").format(
+                        sql.Identifier(table_name),
+                        sql.Identifier(key),
+                        sql.SQL(infer_sql_type(value)),
+                    )
+                )
+
+
+def convert_value(value: Any) -> Any:
+    if isinstance(value, str):
+        try:
+            return datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except Exception:
+            return value
+    return value
+
+
+def insert_record(
+    conn: psycopg2.extensions.connection, table_name: str, record: dict
+) -> None:
+    columns = list(record.keys())
+    values = [convert_value(record[c]) for c in columns]
+    with conn.cursor() as cur:
+        cur.execute(
+            sql.SQL("INSERT INTO {} ({}) VALUES ({})").format(
+                sql.Identifier(table_name),
+                sql.SQL(", ").join(map(sql.Identifier, columns)),
+                sql.SQL(", ").join(sql.Placeholder() * len(columns)),
+            ),
+            values,
+        )

stream2pg/errors.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+
+class ConfigurationError(Exception):
+    pass

stream2pg/sink.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+from typing import Any, Callable, Optional
+
+from .config import ErrorStrategy, from_config
+from .core import create_kafka_stream, create_spark_session, process_batch
+
+
+class Stream2Pg:
+    def __init__(
+        self,
+        config: dict[str, Any],
+        on_metrics: Optional[Callable[..., None]] = None,
+    ):
+        self.config = from_config(config, on_metrics=on_metrics)
+        self.on_metrics = on_metrics
+
+    def run(self) -> None:
+        cfg = self.config
+
+        postgres_cfg = cfg["postgres"]
+        kafka_cfg = cfg["kafka"]
+        processing_cfg = cfg["processing"]
+
+        error_strategy_str = processing_cfg.get("error_strategy", "raise")
+        error_strategy = ErrorStrategy(error_strategy_str)
+
+        checkpoint_location = processing_cfg.get(
+            "checkpoint_location", "./checkpoints/kafka_to_postgres"
+        )
+
+        spark = create_spark_session()
+        spark.sparkContext.setLogLevel("WARN")
+
+        df = create_kafka_stream(spark, kafka_cfg)
+
+        def foreach_batch_fn(batch_df, batch_id):
+            process_batch(
+                batch_df,
+                batch_id,
+                postgres_cfg,
+                error_strategy,
+                on_metrics=self.on_metrics,
+            )
+
+        query = (
+            df.writeStream.foreachBatch(foreach_batch_fn)
+            .option("checkpointLocation", checkpoint_location)
+            .start()
+        )
+        query.awaitTermination()
+
+
+def run(
+    config: dict[str, Any],
+    on_metrics: Optional[Callable[..., None]] = None,
+) -> None:
+    sink = Stream2Pg(config, on_metrics=on_metrics)
+    sink.run()

stream2pg-0.1.0.dist-info/METADATA

@@ -0,0 +1,294 @@
+Metadata-Version: 2.4
+Name: stream2pg
+Version: 0.1.0
+Summary: Kafka to PostgreSQL sink using Spark Structured Streaming
+Author-email: Sahand Akramipour <sahandap@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/shndap/stream2pg
+Project-URL: Documentation, https://github.com/shndap/stream2pg
+Project-URL: Repository, https://github.com/shndap/stream2pg
+Project-URL: Issues, https://github.com/shndap/stream2pg/issues
+Keywords: kafka,spark,postgresql,streaming,etl
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: psycopg2-binary>=2.9
+Requires-Dist: pyspark>=3.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Dynamic: license-file
+
+<div align="center">
+
+# stream2pg
+
+**Automatically stream JSON data from Kafka into PostgreSQL using Spark Structured Streaming.**
+
+Create tables. Evolve schemas. Load data.
+
+<a href="https://pypi.org/project/stream2pg/">
+  <img src="https://img.shields.io/pypi/v/stream2pg.svg" alt="PyPI">
+</a>
+<a href="https://pypi.org/project/stream2pg/">
+  <img src="https://img.shields.io/pypi/pyversions/stream2pg.svg" alt="Python">
+</a>
+<a href="LICENSE">
+  <img src="https://img.shields.io/github/license/YOUR_USERNAME/stream2pg">
+</a>
+
+</div>
+
+---
+
+## Features
+
+- 🚀 Kafka → PostgreSQL streaming
+- 🔄 Automatic schema evolution
+- 📦 Spark Structured Streaming
+- 🐘 PostgreSQL integration
+- 📊 Metrics callbacks
+- ⚡ Minimal configuration
+
+
+
+## How It Works
+
+Given Kafka messages like:
+
+```json
+{
+  "vehicle_id": 42,
+  "speed": 18.7,
+  "timestamp": "2025-01-01T12:00:00Z"
+}
+```
+
+`stream2pg` will:
+
+1. Subscribe to matching Kafka topics
+2. Create PostgreSQL tables automatically
+3. Infer column types
+4. Add new columns when unseen fields appear
+5. Insert records into PostgreSQL
+
+No manual DDL required.
+
+
+
+## Installation
+
+### Development
+
+```bash
+pip install -e .
+```
+
+### From PyPI
+
+```bash
+pip install stream2pg
+```
+
+
+
+## Quick Start
+
+```python
+from stream2pg import Stream2Pg
+
+config = {
+    "postgres": {
+        "host": "localhost",
+        "port": 5432,
+        "dbname": "mobility",
+        "user": "postgres",
+        "password": "secret",
+    },
+    "kafka": {
+        "bootstrap_servers": "localhost:9092",
+        "subscribe_pattern": "mobility-.*",
+        "starting_offsets": "earliest",
+        "fail_on_data_loss": "false",
+    },
+    "processing": {
+        "error_strategy": "SKIP_ON_ERROR",
+        "checkpoint_location": "./checkpoints/kafka_to_postgres",
+    },
+}
+
+sink = Stream2Pg(config)
+sink.run()
+```
+
+
+
+## Metrics
+
+A metrics callback can be provided to observe batch execution.
+
+```python
+def on_metrics(
+    batch_id,
+    row_count,
+    inserted_count,
+    skipped_count,
+    elapsed_ms,
+):
+    print(
+        f"Batch {batch_id}: "
+        f"{inserted_count} inserted, "
+        f"{skipped_count} skipped "
+        f"in {elapsed_ms} ms"
+    )
+
+sink = Stream2Pg(config, on_metrics=on_metrics)
+sink.run()
+```
+
+
+
+## Functional API
+
+If you prefer not to instantiate the class directly:
+
+```python
+from stream2pg import run
+
+run(config)
+```
+
+
+
+## Configuration
+
+### PostgreSQL
+
+```python
+{
+    "host": "localhost",
+    "port": 5432,
+    "dbname": "mobility",
+    "user": "postgres",
+    "password": "secret",
+}
+```
+
+### Kafka
+
+```python
+{
+    "bootstrap_servers": "localhost:9092",
+    "subscribe_pattern": "mobility-.*",
+    "starting_offsets": "earliest",
+    "fail_on_data_loss": "false",
+}
+```
+
+### Processing
+
+```python
+{
+    "error_strategy": "SKIP_ON_ERROR",
+    "checkpoint_location": "./checkpoints/kafka_to_postgres",
+}
+```
+
+
+
+## Error Handling
+
+### `RAISE`
+
+Fail the current batch immediately when an error occurs.
+
+```python
+"error_strategy": "RAISE"
+```
+
+### `SKIP_ON_ERROR`
+
+Skip invalid records and continue processing the remaining batch.
+
+```python
+"error_strategy": "SKIP_ON_ERROR"
+```
+
+
+
+## API Reference
+
+### `Stream2Pg`
+
+```python
+Stream2Pg(config, on_metrics=None)
+```
+
+Creates a streaming sink instance.
+
+Parameters:
+
+| Parameter    | Description               |
+|  | - |
+| `config`     | Configuration dictionary  |
+| `on_metrics` | Optional metrics callback |
+
+
+
+### `Stream2Pg.run()`
+
+Starts the Kafka → PostgreSQL streaming pipeline.
+
+```python
+sink.run()
+```
+
+
+
+### `run()`
+
+Convenience wrapper around `Stream2Pg`.
+
+```python
+from stream2pg import run
+
+run(config)
+```
+
+
+
+### Exceptions
+
+#### `ConfigurationError`
+
+Raised when the provided configuration is invalid.
+
+```python
+from stream2pg.errors import ConfigurationError
+```
+
+
+
+## Requirements
+
+* Python 3.9+
+* Apache Spark
+* Kafka
+* PostgreSQL
+
+
+
+## License
+
+[MIT](./LICENSE)

stream2pg-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+stream2pg/__init__.py,sha256=ZkQc0w7vhOdFPYjh6dAs-ch6Sgbwcd-BD2bzR2d_TKc,98
+stream2pg/config.py,sha256=YupmwD8aAPmckp2joj2-pINLRMt-_p1t_7m0jXxQadk,755
+stream2pg/core.py,sha256=crxuiWuNZbn6YamL6ST-eChbyOoUHHocS1otI7ZeVj4,3292
+stream2pg/db.py,sha256=PZlcXlws23DmsLMgMLf5niLjtbYO_u4cy-PGQnTFPGI,2475
+stream2pg/errors.py,sha256=rlRy0gR3wUULpMwLN4T0K2XEz0fA9e1yBQyI4XDukFE,83
+stream2pg/sink.py,sha256=q8YbyZ6HfRbz_ukKcuDZHRxLpgq08CR5LswaLjQAMag,1650
+stream2pg-0.1.0.dist-info/licenses/LICENSE,sha256=fc2MP3yg9KGEIFxFo8vDYQlRKACFnEnaRmHT03OJsFo,1074
+stream2pg-0.1.0.dist-info/METADATA,sha256=skvzVqyaErQbTSwC6iQj3A-KcG7M75xQ_z4sQHRpYfs,5473
+stream2pg-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+stream2pg-0.1.0.dist-info/top_level.txt,sha256=aVAVSZVoY2i4p2og17v4kHHb-Dy2nSroV0ana5_q4xQ,10
+stream2pg-0.1.0.dist-info/RECORD,,

stream2pg-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

stream2pg-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sahand Akramipour
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

stream2pg-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+stream2pg