killuhub 0.1.0__py3-none-any.whl

killuhub/__init__.py

@@ -0,0 +1,74 @@
+"""
+KilluHub — Pluggable data ingestion framework.
+
+Sources → Connectors → Engine (Spark/Flink) → Bronze → Silver → Iceberg
+
+Quick start (raw pipeline):
+    from killuhub import Pipeline, PipelineConfig, ConnectorConfig
+
+    Pipeline(PipelineConfig(
+        connector_name="postgres",
+        connector_config=ConnectorConfig.from_dict({...}),
+        target_table="local.db.orders",
+    )).run()
+
+Medallion (Bronze → Silver):
+    from killuhub import BronzePipeline, BronzeConfig, SilverPipeline, SilverConfig
+    from killuhub.core.batch import BatchMode, BatchConfig
+
+    BronzePipeline(BronzeConfig(
+        pipeline_config=PipelineConfig(...),
+        batch_config=BatchConfig(mode=BatchMode.FULL),
+        bronze_table="local.bronze.orders",
+        source_name="postgres.shop.orders",
+    )).run()
+
+    SilverPipeline(SilverConfig(
+        bronze_table="local.bronze.orders",
+        silver_table="local.silver.orders",
+        batch_config=BatchConfig(mode=BatchMode.INCREMENTAL),
+        key_columns=["order_id"],
+        date_columns=["created_at"],
+    )).run()
+"""
+# Import subpackages so auto-registration runs
+import killuhub.connectors  # noqa: F401
+import killuhub.processing  # noqa: F401
+
+from killuhub.core import (
+    BaseConnector,
+    BaseEngine,
+    BaseStorageWriter,
+    ConnectorConfig,
+    PipelineConfig,
+    BatchMode,
+    BatchConfig,
+    IncrementalState,
+    default_registry,
+)
+from killuhub.ingestion import Pipeline, PipelineScheduler
+from killuhub.layers import BronzePipeline, BronzeConfig, SilverPipeline, SilverConfig
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Raw pipeline
+    "Pipeline",
+    "PipelineScheduler",
+    "PipelineConfig",
+    "ConnectorConfig",
+    # Medallion layers
+    "BronzePipeline",
+    "BronzeConfig",
+    "SilverPipeline",
+    "SilverConfig",
+    # Batch primitives
+    "BatchMode",
+    "BatchConfig",
+    "IncrementalState",
+    # Extension points
+    "BaseConnector",
+    "BaseEngine",
+    "BaseStorageWriter",
+    "default_registry",
+]

killuhub/connectors/__init__.py

@@ -0,0 +1,18 @@
+from killuhub.connectors.postgres.connector import PostgresConnector
+from killuhub.connectors.mysql.connector import MySQLConnector
+from killuhub.connectors.kafka.connector import KafkaConnector
+from killuhub.connectors.rest_api.connector import RestApiConnector
+from killuhub.core.registry import default_registry
+
+# Auto-register all built-in connectors
+default_registry.register_connector("postgres", PostgresConnector)
+default_registry.register_connector("mysql", MySQLConnector)
+default_registry.register_connector("kafka", KafkaConnector)
+default_registry.register_connector("rest_api", RestApiConnector)
+
+__all__ = [
+    "PostgresConnector",
+    "MySQLConnector",
+    "KafkaConnector",
+    "RestApiConnector",
+]

killuhub/connectors/kafka/__init__.py

@@ -0,0 +1,3 @@
+from killuhub.connectors.kafka.connector import KafkaConnector
+
+__all__ = ["KafkaConnector"]

killuhub/connectors/kafka/connector.py

@@ -0,0 +1,88 @@
+"""
+Kafka connector for KilluHub.
+
+Required config keys:
+    bootstrap_servers  — comma-separated broker list
+    topic              — topic to consume
+    group_id           — consumer group id
+
+Optional:
+    auto_offset_reset  (default "earliest")
+    max_records        (int)  — stop after N records (useful for batch jobs)
+    poll_timeout_ms    (int, default 1000)
+    value_deserializer ("json" | "string", default "json")
+"""
+import json
+from typing import Any, Iterator
+
+from killuhub.core.connector_interface import BaseConnector
+from killuhub.core.config import ConnectorConfig
+
+
+class KafkaConnector(BaseConnector):
+    def __init__(self, config: ConnectorConfig):
+        super().__init__(config)
+        self._consumer = None
+
+    def connect(self) -> None:
+        try:
+            from confluent_kafka import Consumer
+        except ImportError as e:
+            raise ImportError(
+                "confluent-kafka is required for KafkaConnector. "
+                "Install it with: pip install confluent-kafka"
+            ) from e
+
+        from confluent_kafka import Consumer
+
+        self._consumer = Consumer(
+            {
+                "bootstrap.servers": self.config.require("bootstrap_servers"),
+                "group.id": self.config.require("group_id"),
+                "auto.offset.reset": self.config.get("auto_offset_reset", "earliest"),
+                "enable.auto.commit": False,
+            }
+        )
+        self._consumer.subscribe([self.config.require("topic")])
+        self._connected = True
+
+    def extract(self) -> Iterator[dict[str, Any]]:
+        if not self._connected:
+            self.connect()
+
+        max_records = self.config.get("max_records", None)
+        poll_timeout = self.config.get("poll_timeout_ms", 1_000) / 1_000
+        deserializer = self.config.get("value_deserializer", "json")
+        count = 0
+
+        while True:
+            msg = self._consumer.poll(timeout=poll_timeout)
+
+            if msg is None:
+                # No message within timeout — treat as end of stream for batch mode
+                break
+
+            if msg.error():
+                from confluent_kafka import KafkaError
+                if msg.error().code() == KafkaError._PARTITION_EOF:
+                    break
+                raise RuntimeError(f"Kafka error: {msg.error()}")
+
+            raw = msg.value()
+            if deserializer == "json":
+                record = json.loads(raw.decode("utf-8"))
+            else:
+                record = {"value": raw.decode("utf-8")}
+
+            yield record
+            self._consumer.commit(message=msg)
+            count += 1
+
+            if max_records and count >= max_records:
+                break
+
+    def close(self) -> None:
+        if self._consumer:
+            self._consumer.close()
+            self._consumer = None
+            self._connected = False

killuhub/connectors/mysql/__init__.py

@@ -0,0 +1,3 @@
+from killuhub.connectors.mysql.connector import MySQLConnector
+
+__all__ = ["MySQLConnector"]

killuhub/connectors/mysql/connector.py

@@ -0,0 +1,86 @@
+"""
+MySQL connector for KilluHub.
+
+Uses mysql-connector-python with a server-side cursor (buffered=False) so large
+tables are streamed row-by-row without loading the full result set into memory.
+
+Required config keys:
+    host, database, user, password, query
+
+Optional:
+    port       (int,  default 3306)
+    batch_size (int,  default 1000) — rows fetched per round-trip
+    ssl_ca     (str)  — path to CA certificate for TLS connections
+    ssl_cert   (str)  — path to client certificate
+    ssl_key    (str)  — path to client private key
+"""
+from typing import Any, Iterator
+
+from killuhub.core.connector_interface import BaseConnector
+from killuhub.core.config import ConnectorConfig
+
+
+class MySQLConnector(BaseConnector):
+    def __init__(self, config: ConnectorConfig):
+        super().__init__(config)
+        self._conn = None
+
+    def connect(self) -> None:
+        try:
+            import mysql.connector
+        except ImportError as e:
+            raise ImportError(
+                "mysql-connector-python is required for MySQLConnector. "
+                "Install it with: pip install mysql-connector-python"
+            ) from e
+
+        kwargs: dict[str, Any] = {
+            "host": self.config.require("host"),
+            "port": self.config.get("port", 3306),
+            "database": self.config.require("database"),
+            "user": self.config.require("user"),
+            "password": self.config.require("password"),
+            # consume_results=True allows re-use of the connection after
+            # the cursor is fully iterated without explicitly fetching all rows
+            "consume_results": True,
+        }
+
+        ssl_ca = self.config.get("ssl_ca")
+        if ssl_ca:
+            kwargs["ssl_ca"] = ssl_ca
+            kwargs["ssl_verify_cert"] = True
+        ssl_cert = self.config.get("ssl_cert")
+        if ssl_cert:
+            kwargs["ssl_cert"] = ssl_cert
+            kwargs["ssl_key"] = self.config.require("ssl_key")
+
+        import mysql.connector
+        self._conn = mysql.connector.connect(**kwargs)
+        self._connected = True
+
+    def extract(self) -> Iterator[dict[str, Any]]:
+        if not self._connected:
+            self.connect()
+
+        query = self.config.require("query")
+        batch_size = self.config.get("batch_size", 1_000)
+
+        # buffered=False → server-side streaming cursor, safe for large tables.
+        # fetchmany() pulls batch_size rows per round-trip to bound memory usage.
+        cursor = self._conn.cursor(buffered=False, dictionary=True)
+        try:
+            cursor.execute(query)
+            while True:
+                rows = cursor.fetchmany(batch_size)
+                if not rows:
+                    break
+                for row in rows:
+                    yield dict(row)
+        finally:
+            cursor.close()
+
+    def close(self) -> None:
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+            self._connected = False

killuhub/connectors/postgres/__init__.py

@@ -0,0 +1,3 @@
+from killuhub.connectors.postgres.connector import PostgresConnector
+
+__all__ = ["PostgresConnector"]

killuhub/connectors/postgres/connector.py

@@ -0,0 +1,59 @@
+"""
+PostgreSQL connector for KilluHub.
+
+Required config keys:
+    host, port, database, user, password, query
+
+Optional:
+    batch_size (int, default 1000) — rows fetched per server-side cursor fetch
+"""
+from typing import Any, Iterator
+
+from killuhub.core.connector_interface import BaseConnector
+from killuhub.core.config import ConnectorConfig
+
+
+class PostgresConnector(BaseConnector):
+    def __init__(self, config: ConnectorConfig):
+        super().__init__(config)
+        self._conn = None
+        self._cursor = None
+
+    def connect(self) -> None:
+        try:
+            import psycopg2
+        except ImportError as e:
+            raise ImportError(
+                "psycopg2 is required for PostgresConnector. "
+                "Install it with: pip install psycopg2-binary"
+            ) from e
+
+        self._conn = psycopg2.connect(
+            host=self.config.require("host"),
+            port=self.config.get("port", 5432),
+            dbname=self.config.require("database"),
+            user=self.config.require("user"),
+            password=self.config.require("password"),
+        )
+        self._connected = True
+
+    def extract(self) -> Iterator[dict[str, Any]]:
+        if not self._connected:
+            self.connect()
+
+        query = self.config.require("query")
+        batch_size = self.config.get("batch_size", 1_000)
+
+        # Named cursor enables server-side pagination — safe for large tables
+        with self._conn.cursor(name="killuhub_cursor") as cur:
+            cur.itersize = batch_size
+            cur.execute(query)
+            columns = [desc[0] for desc in cur.description]
+            for row in cur:
+                yield dict(zip(columns, row))
+
+    def close(self) -> None:
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+            self._connected = False

killuhub/connectors/rest_api/__init__.py

@@ -0,0 +1,3 @@
+from killuhub.connectors.rest_api.connector import RestApiConnector
+
+__all__ = ["RestApiConnector"]

killuhub/connectors/rest_api/connector.py

@@ -0,0 +1,167 @@
+"""
+REST API connector for KilluHub.
+
+Handles paginated JSON APIs with configurable pagination strategies.
+
+Required config keys:
+    url   — base endpoint URL
+
+Optional:
+    method          ("GET" | "POST", default "GET")
+    headers         (dict)
+    params          (dict) — query string params
+    body            (dict) — request body for POST
+    auth_type       ("bearer" | "basic" | None)
+    auth_token      — used with auth_type "bearer"
+    auth_user       — used with auth_type "basic"
+    auth_password   — used with auth_type "basic"
+    data_key        — JSON key that holds the records list (e.g. "data", "results")
+    pagination      ("none" | "page" | "cursor" | "offset", default "none")
+    page_param      — query param name for page number (default "page")
+    page_size_param — query param name for page size (default "page_size")
+    page_size       (int, default 100)
+    cursor_key      — JSON key that holds next cursor (default "next_cursor")
+    cursor_param    — query param name for cursor (default "cursor")
+    max_pages       (int) — safety cap on pagination depth
+"""
+from typing import Any, Iterator
+
+from killuhub.core.connector_interface import BaseConnector
+from killuhub.core.config import ConnectorConfig
+
+
+class RestApiConnector(BaseConnector):
+    def __init__(self, config: ConnectorConfig):
+        super().__init__(config)
+        self._session = None
+
+    def connect(self) -> None:
+        try:
+            import requests
+        except ImportError as e:
+            raise ImportError(
+                "requests is required for RestApiConnector. "
+                "Install it with: pip install requests"
+            ) from e
+
+        import requests
+        self._session = requests.Session()
+
+        headers = self.config.get("headers", {})
+        auth_type = self.config.get("auth_type")
+
+        if auth_type == "bearer":
+            headers["Authorization"] = f"Bearer {self.config.require('auth_token')}"
+        elif auth_type == "basic":
+            from requests.auth import HTTPBasicAuth
+            self._session.auth = HTTPBasicAuth(
+                self.config.require("auth_user"),
+                self.config.require("auth_password"),
+            )
+
+        self._session.headers.update(headers)
+        self._connected = True
+
+    def extract(self) -> Iterator[dict[str, Any]]:
+        if not self._connected:
+            self.connect()
+
+        pagination = self.config.get("pagination", "none")
+
+        if pagination == "page":
+            yield from self._paginate_by_page()
+        elif pagination == "cursor":
+            yield from self._paginate_by_cursor()
+        elif pagination == "offset":
+            yield from self._paginate_by_offset()
+        else:
+            yield from self._fetch_single()
+
+    def _request(self, params: dict | None = None) -> Any:
+        method = self.config.get("method", "GET").upper()
+        url = self.config.require("url")
+        base_params = dict(self.config.get("params") or {})
+        if params:
+            base_params.update(params)
+
+        if method == "GET":
+            resp = self._session.get(url, params=base_params)
+        else:
+            resp = self._session.post(
+                url, params=base_params, json=self.config.get("body")
+            )
+        resp.raise_for_status()
+        return resp.json()
+
+    def _extract_records(self, data: Any) -> list[dict[str, Any]]:
+        data_key = self.config.get("data_key")
+        if data_key:
+            return data.get(data_key, [])
+        if isinstance(data, list):
+            return data
+        return [data]
+
+    def _fetch_single(self) -> Iterator[dict[str, Any]]:
+        yield from self._extract_records(self._request())
+
+    def _paginate_by_page(self) -> Iterator[dict[str, Any]]:
+        page_param = self.config.get("page_param", "page")
+        size_param = self.config.get("page_size_param", "page_size")
+        page_size = self.config.get("page_size", 100)
+        max_pages = self.config.get("max_pages", None)
+        page = 1
+
+        while True:
+            data = self._request({page_param: page, size_param: page_size})
+            records = self._extract_records(data)
+            if not records:
+                break
+            yield from records
+            page += 1
+            if max_pages and page > max_pages:
+                break
+
+    def _paginate_by_cursor(self) -> Iterator[dict[str, Any]]:
+        cursor_key = self.config.get("cursor_key", "next_cursor")
+        cursor_param = self.config.get("cursor_param", "cursor")
+        max_pages = self.config.get("max_pages", None)
+        cursor = None
+        page = 0
+
+        while True:
+            params = {cursor_param: cursor} if cursor else {}
+            data = self._request(params)
+            records = self._extract_records(data)
+            if not records:
+                break
+            yield from records
+            cursor = data.get(cursor_key)
+            if not cursor:
+                break
+            page += 1
+            if max_pages and page >= max_pages:
+                break
+
+    def _paginate_by_offset(self) -> Iterator[dict[str, Any]]:
+        size_param = self.config.get("page_size_param", "limit")
+        page_size = self.config.get("page_size", 100)
+        max_pages = self.config.get("max_pages", None)
+        offset = 0
+        page = 0
+
+        while True:
+            data = self._request({"offset": offset, size_param: page_size})
+            records = self._extract_records(data)
+            if not records:
+                break
+            yield from records
+            offset += len(records)
+            page += 1
+            if max_pages and page >= max_pages:
+                break
+
+    def close(self) -> None:
+        if self._session:
+            self._session.close()
+            self._session = None
+            self._connected = False

killuhub/core/__init__.py

@@ -0,0 +1,36 @@
+from killuhub.core.connector_interface import BaseConnector
+from killuhub.core.engine_interface import BaseEngine
+from killuhub.core.storage_interface import BaseStorageWriter
+from killuhub.core.config import ConnectorConfig, PipelineConfig
+from killuhub.core.registry import Registry, default_registry
+from killuhub.core.exceptions import (
+    KilluHubError, BatchModeError, IncrementalStateError, ContractViolationError
+)
+from killuhub.core.batch import ExecutionMode, BatchMode, BatchConfig, StreamingConfig, IncrementalState
+from killuhub.core.contract import ContractSpec, ColumnSpec, ContractValidator, ContractReport
+from killuhub.core.environment import RuntimeEnvironment, detect as detect_environment
+
+__all__ = [
+    "BaseConnector",
+    "BaseEngine",
+    "BaseStorageWriter",
+    "ConnectorConfig",
+    "PipelineConfig",
+    "Registry",
+    "default_registry",
+    "KilluHubError",
+    "BatchModeError",
+    "IncrementalStateError",
+    "ContractViolationError",
+    "ExecutionMode",
+    "BatchMode",
+    "BatchConfig",
+    "StreamingConfig",
+    "IncrementalState",
+    "ContractSpec",
+    "ColumnSpec",
+    "ContractValidator",
+    "ContractReport",
+    "RuntimeEnvironment",
+    "detect_environment",
+]

killuhub/core/batch.py

@@ -0,0 +1,135 @@
+"""
+Batch and streaming processing primitives for KilluHub.
+
+ExecutionMode controls whether a pipeline runs as a one-shot batch job or as a
+continuous stream. Within batch mode, BatchMode controls whether all records are
+processed (FULL) or only records added since the last run (INCREMENTAL).
+
+These are core-layer types — no Spark, no I/O, no external dependencies.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class ExecutionMode(str, Enum):
+    """
+    Top-level pipeline execution mode.
+
+    BATCH     — one-shot job: read, transform, write, exit.
+    STREAMING — continuous job: run indefinitely, processing records as they
+                arrive (Spark Structured Streaming or Flink).
+    """
+    BATCH     = "batch"
+    STREAMING = "streaming"
+
+
+class BatchMode(str, Enum):
+    """
+    Strategy for a BATCH execution run.
+
+    Using `str` as a mixin base means `BatchMode.FULL == "full"` is True
+    and the value serialises to a plain string in JSON state files without
+    a custom encoder.
+
+    FULL        — read/process all available records.
+    INCREMENTAL — read only records newer than the last watermark cursor.
+    """
+    FULL        = "full"
+    INCREMENTAL = "incremental"
+
+
+@dataclass
+class BatchConfig:
+    """
+    Batch execution parameters — only used when ExecutionMode is BATCH.
+
+    Attributes:
+        mode:
+            FULL — read/process all available records.
+            INCREMENTAL — read only records newer than the last watermark.
+
+        watermark_column:
+            Column used to advance the incremental cursor.
+            For Bronze this is typically the source's own timestamp column
+            (e.g. `updated_at`).
+            For Silver this is typically `_ingested_at` (the Bronze metadata
+            column) so Silver can filter on a column it controls regardless
+            of the source schema.
+
+        initial_watermark:
+            ISO-8601 string used as the lower bound on the very first
+            INCREMENTAL run when no prior state exists.
+            Default epoch means "process everything on first run".
+
+        batch_id:
+            Unique identifier for this run, written to the `_batch_id`
+            metadata column in Bronze. If empty, Bronze auto-generates a UUID
+            at runtime. Injecting a deterministic ID makes reruns idempotent.
+    """
+    mode: BatchMode
+    watermark_column: str = "updated_at"
+    initial_watermark: str = "1970-01-01T00:00:00"
+    batch_id: str = ""
+
+
+@dataclass
+class StreamingConfig:
+    """
+    Streaming execution parameters — only used when ExecutionMode is STREAMING.
+
+    Attributes:
+        trigger:
+            Spark Structured Streaming trigger type.
+            processingTime  — microbatch, runs every `trigger_interval`.
+            once            — processes all available data in one microbatch then stops.
+            availableNow    — like once but respects rate limits.
+            continuous      — experimental low-latency continuous processing.
+
+        trigger_interval:
+            Microbatch interval string (e.g. "30 seconds", "1 minute").
+            Only used when trigger is processingTime.
+
+        checkpoint_location:
+            Path where Spark stores streaming checkpoint state.
+            Must be durable storage (S3, HDFS, ADLS) in production.
+            Required for exactly-once guarantees and job restarts.
+
+        output_mode:
+            append   — only new rows written each microbatch (default, Iceberg).
+            update   — rows that changed are written (requires watermarking).
+            complete — full result written each time (for aggregations).
+    """
+    trigger: str = "processingTime"
+    trigger_interval: str = "30 seconds"
+    checkpoint_location: str = "/tmp/killuhub-checkpoints"
+    output_mode: str = "append"
+
+
+@dataclass
+class IncrementalState:
+    """
+    Persisted watermark state for an INCREMENTAL Silver run.
+
+    Each (source_table, target_table) pair has one state record.
+    After a successful Silver run the `last_watermark` is updated to
+    max(watermark_column) observed in that batch, so the next run
+    starts exactly from where this one ended.
+
+    Attributes:
+        source_table:    Fully qualified bronze Iceberg table name.
+        target_table:    Fully qualified silver Iceberg table name.
+        last_watermark:  ISO-8601 string. Next incremental run will
+                         filter `watermark_column > last_watermark`.
+        updated_at:      ISO-8601 string of when this state was written.
+                         Informational — not used for query filtering.
+    """
+    source_table: str
+    target_table: str
+    last_watermark: str
+    updated_at: str = field(default="")
+
+    def state_key(self) -> str:
+        """Stable dict key for JSON serialisation."""
+        return f"{self.source_table}::{self.target_table}"