datafun-streaming 0.1.0__py3-none-any.whl

datafun_streaming/kafka/kafka_admin_utils.py

@@ -0,0 +1,211 @@
+"""src/datafun_streaming/kafka/kafka_admin_utils.py.
+
+Kafka topic management helpers for streaming examples.
+"""
+
+# === IMPORTS ===
+
+import time
+
+from confluent_kafka import Consumer, TopicPartition
+from confluent_kafka.admin import AdminClient
+from confluent_kafka.cimpl import KafkaException, NewTopic
+
+from datafun_streaming.kafka.errors import kafka_admin_failed_message
+from datafun_streaming.kafka.kafka_settings import KafkaSettings
+
+# === EXPORTS
+
+__all__ = [
+    "create_admin_client",
+    "create_topic",
+    "delete_topic",
+    "list_topics",
+    "topic_exists",
+    "get_topic_message_count",
+]
+
+# === DECLARE CONSTANTS ===
+
+# rdkafka's internal broker handshake takes up to ~4 seconds on Windows
+# before the AdminClient is ready to accept calls. These constants
+# control how long we wait and how many times we retry.
+ADMIN_READY_RETRIES: int = 5
+ADMIN_READY_DELAY_SECONDS: float = 2.0
+
+
+# === DEFINE ADMIN HELPER FUNCTIONS ===
+
+
+def create_admin_client(settings: KafkaSettings) -> AdminClient:
+    """Create a Kafka AdminClient."""
+    return AdminClient({"bootstrap.servers": settings.bootstrap_servers})
+
+
+def create_topic(
+    admin: AdminClient,
+    topic: str,
+    *,
+    num_partitions: int = 1,
+    replication_factor: int = 1,
+) -> None:
+    """Create a Kafka topic if it does not already exist.
+
+    Arguments:
+        admin: An AdminClient instance.
+        topic: The topic name to create.
+        num_partitions: Number of partitions (default 1 for local dev).
+        replication_factor: Replication factor (default 1 for local dev).
+
+    Raises:
+        RuntimeError: If topic creation fails.
+    """
+    if topic_exists(admin, topic):
+        return
+
+    new_topic = NewTopic(
+        topic,
+        num_partitions=num_partitions,
+        replication_factor=replication_factor,
+    )
+
+    futures = admin.create_topics([new_topic])
+
+    for topic_name, future in futures.items():
+        try:
+            future.result()
+        except KafkaException as error:
+            msg = (
+                f"Failed to create topic {topic_name!r}.\n"
+                f"Kafka reported: {error}\n\n"
+                "Check that Kafka is running and that you have permission to create topics."
+            )
+            raise RuntimeError(msg) from error
+
+
+def delete_topic(admin: AdminClient, topic: str) -> None:
+    """Delete a Kafka topic if it exists.
+
+    Deleting a topic removes all its messages. Run the producer again
+    after deleting to repopulate the topic.
+
+    Arguments:
+        admin: An AdminClient instance.
+        topic: The topic name to delete.
+
+    Raises:
+        RuntimeError: If topic deletion fails.
+    """
+    if not topic_exists(admin, topic):
+        return
+
+    futures = admin.delete_topics([topic])
+
+    for topic_name, future in futures.items():
+        try:
+            future.result()
+        except KafkaException as error:
+            msg = (
+                f"Failed to delete topic {topic_name!r}.\n"
+                f"Kafka reported: {error}\n\n"
+                "Check that Kafka is running and that you have permission to delete topics."
+            )
+            raise RuntimeError(msg) from error
+
+
+def list_topics(admin: AdminClient) -> list[str]:
+    """Return a sorted list of topic names currently in Kafka.
+
+    Retries several times to allow rdkafka's broker handshake to complete.
+    On Windows, the handshake can take up to ~4 seconds after the AdminClient
+    is created, which causes an immediate call to fail with a transport error.
+
+    Arguments:
+        admin: An AdminClient instance.
+
+    Returns:
+        A sorted list of topic name strings.
+
+    Raises:
+        RuntimeError: If Kafka is unreachable after all retries.
+    """
+    last_error: Exception | None = None
+
+    for attempt in range(1, ADMIN_READY_RETRIES + 1):
+        try:
+            metadata = admin.list_topics(timeout=5)
+            return sorted(metadata.topics.keys())
+        except KafkaException as error:
+            last_error = error
+            if attempt < ADMIN_READY_RETRIES:
+                time.sleep(ADMIN_READY_DELAY_SECONDS)
+
+    msg = kafka_admin_failed_message(
+        operation="list_topics",
+        topic="(all)",
+        detail=(
+            f"Kafka did not respond after {ADMIN_READY_RETRIES} attempts.\n"
+            f"    Last error: {last_error}"
+        ),
+    )
+    raise RuntimeError(msg) from last_error
+
+
+def topic_exists(admin: AdminClient, topic: str) -> bool:
+    """Return True if the topic already exists in Kafka."""
+    return topic in list_topics(admin)
+
+
+def get_topic_message_count(
+    admin: AdminClient, topic: str, settings: KafkaSettings
+) -> int:
+    """Return the total number of messages available in a topic.
+
+    Sums the high-water offset across all partitions.
+    This reflects the total messages ever produced to the topic,
+    not the number of unread messages for a specific consumer group.
+
+    Arguments:
+        admin: An AdminClient instance.
+        topic: The topic name to inspect.
+        settings: KafkaSettings instance containing configuration.
+
+    Returns:
+        Total message count across all partitions, or 0 if topic is empty.
+
+    Raises:
+        RuntimeError: If topic metadata cannot be retrieved.
+    """
+    try:
+        metadata = admin.list_topics(topic=topic, timeout=5)
+    except KafkaException as error:
+        msg = kafka_admin_failed_message(
+            operation="list_topics",
+            topic=topic,
+            detail=str(error),
+        )
+        raise RuntimeError(msg) from error
+
+    topic_metadata = metadata.topics.get(topic)
+    if topic_metadata is None:
+        return 0
+
+    bootstrap_servers = settings.bootstrap_servers
+    temp_consumer = Consumer(
+        {
+            "bootstrap.servers": bootstrap_servers,
+            "group.id": "_offset_inspector",
+            "enable.auto.commit": "false",
+        }
+    )
+
+    total = 0
+    try:
+        for partition_id in topic_metadata.partitions:
+            tp = TopicPartition(topic, partition_id)
+            low, high = temp_consumer.get_watermark_offsets(tp, timeout=5)
+            total += max(0, high - low)
+    finally:
+        temp_consumer.close()
+
+    return total

datafun_streaming/kafka/kafka_connection_utils.py

@@ -0,0 +1,46 @@
+"""src/datafun_streaming/kafka/kafka_connection_utils.py.
+
+Kafka connection helpers for streaming examples.
+"""
+
+# === IMPORTS ===
+
+import socket
+
+from datafun_streaming.kafka.errors import kafka_not_reachable_message
+from datafun_streaming.kafka.kafka_settings import KafkaSettings
+
+# === EXPORTS ===
+
+__all__ = [
+    "verify_kafka_connection",
+]
+
+
+def verify_kafka_connection(settings: KafkaSettings) -> None:
+    """Verify that the Kafka bootstrap server is reachable."""
+    bootstrap_server = settings.bootstrap_servers.split(",")[0].strip()
+
+    if ":" not in bootstrap_server:
+        msg = (
+            "KAFKA_BOOTSTRAP_SERVERS must include host and port, "
+            f"but got {bootstrap_server!r}."
+        )
+        raise ConnectionError(msg)
+
+    host, port_text = bootstrap_server.rsplit(":", 1)
+
+    try:
+        port = int(port_text)
+    except ValueError as error:
+        msg = f"KAFKA_BOOTSTRAP_SERVERS has an invalid port. Got {bootstrap_server!r}."
+        raise ConnectionError(msg) from error
+
+    try:
+        with socket.create_connection((host, port), timeout=5):
+            return
+    except OSError as error:
+        msg = kafka_not_reachable_message(
+            bootstrap_servers=settings.bootstrap_servers,
+        )
+        raise ConnectionError(msg) from error

datafun_streaming/kafka/kafka_consumer_utils.py

@@ -0,0 +1,62 @@
+"""src/datafun_streaming/kafka/kafka_consumer_utils.py.
+
+Consumer helpers for Kafka messages.
+"""
+
+# === IMPORTS ===
+
+import logging
+from typing import Any
+
+from confluent_kafka import Consumer
+
+from datafun_streaming.io.io_utils import row_from_json
+from datafun_streaming.kafka.errors import kafka_consume_failed_message
+from datafun_streaming.kafka.kafka_settings import KafkaSettings
+
+# === EXPORTS
+
+__all__ = [
+    "create_consumer",
+    "consume_kafka_message",
+]
+
+# === DEFINE HELPER FUNCTIONS ===
+
+
+def create_consumer(settings: KafkaSettings) -> Consumer:
+    """Create a Kafka consumer."""
+    return Consumer(
+        settings.consumer_config(),
+        logger=logging.getLogger("rdkafka.consumer"),
+    )
+
+
+def consume_kafka_message(
+    *,
+    consumer: Any,
+    timeout_seconds: float,
+) -> dict[str, Any] | None:
+    """Consume one Kafka message and return it as a row dictionary."""
+    message = consumer.poll(timeout_seconds)
+
+    if message is None:
+        return None
+
+    if message.error():
+        msg = kafka_consume_failed_message(detail=str(message.error()))
+        raise RuntimeError(msg)
+
+    raw_value = message.value()
+
+    if raw_value is None:
+        return None
+
+    row = row_from_json(raw_value.decode("utf-8"))
+
+    raw_key = message.key()
+    row["_kafka_key"] = raw_key.decode("utf-8") if raw_key else ""
+    row["_kafka_partition"] = message.partition()
+    row["_kafka_offset"] = message.offset()
+
+    return row

datafun_streaming/kafka/kafka_producer_utils.py

@@ -0,0 +1,96 @@
+"""src/datafun_streaming/kafka/kafka_producer_utils.py.
+
+Kafka producer helpers for streaming examples.
+"""
+
+# === IMPORTS ===
+
+import logging
+from typing import Any
+
+from confluent_kafka import Producer
+
+from datafun_streaming.io.io_utils import row_to_json
+from datafun_streaming.kafka.errors import (
+    kafka_delivery_failed_message,
+)
+from datafun_streaming.kafka.kafka_settings import KafkaSettings
+
+# === EXPORTS ===
+
+__all__ = [
+    "create_producer",
+    "produce_kafka_message",
+]
+
+# === DEFINE HELPER FUNCTIONS ===
+
+
+def create_producer(settings: KafkaSettings) -> Producer:
+    """Create a Kafka producer.
+
+    Arguments:
+        settings: KafkaSettings object with producer configuration.
+
+    Returns:
+        A confluent_kafka.Producer instance.
+    """
+    return Producer(
+        settings.producer_config(),
+        logger=logging.getLogger("rdkafka.producer"),
+    )
+
+
+def produce_kafka_message(
+    *,
+    producer: Producer,
+    topic: str,
+    key: str,
+    message: dict[str, Any],
+) -> None:
+    """Produce one dictionary message to Kafka as JSON.
+
+    Arguments:
+        *: All arguments after the asterisk must be passed as keyword arguments.
+        producer: A confluent_kafka.Producer instance.
+        topic: The Kafka topic to produce to.
+        key: The Kafka message key.
+        message: The message dictionary to produce.
+
+    Returns:
+        None
+
+    Raises:
+        RuntimeError: If Kafka reports a delivery failure.
+
+    This function encodes the message as JSON and produces it to Kafka with the given key.
+    It uses a delivery callback to check for delivery errors and raises a RuntimeError if any occur.
+
+    """
+    delivery_errors: list[str] = []
+
+    def delivery_report(error: Any, delivered_message: Any) -> None:
+        """Record Kafka delivery failure details."""
+        if error is not None:
+            delivery_errors.append(str(error))
+
+    producer.produce(
+        topic=topic,
+        key=key.encode("utf-8"),
+        value=row_to_json(message).encode("utf-8"),
+        callback=delivery_report,
+    )
+
+    producer.poll(0)
+
+    remaining = producer.flush(timeout=10)
+
+    if remaining > 0:
+        detail = f"{remaining} Kafka message(s) were not delivered before timeout."
+        msg = kafka_delivery_failed_message(detail=detail)
+        raise RuntimeError(msg)
+
+    if delivery_errors:
+        detail = "; ".join(delivery_errors)
+        msg = kafka_delivery_failed_message(detail=detail)
+        raise RuntimeError(msg)

datafun_streaming/kafka/kafka_settings.py

@@ -0,0 +1,79 @@
+"""src/datafun_streaming/kafka/kafka_settings.py.
+
+Kafka settings for producer and consumer examples.
+"""
+
+# === IMPORTS ===
+
+from dataclasses import dataclass
+import os
+from typing import Self
+
+from dotenv import load_dotenv
+
+# === EXPORTS ===
+
+__all__ = [
+    "KafkaSettings",
+    "DEFAULT_AUTO_OFFSET_RESET",
+    "DEFAULT_BOOTSTRAP_SERVERS",
+    "DEFAULT_GROUP_ID",
+    "DEFAULT_TOPIC",
+]
+
+# === DECLARE DEFAULTS ===
+
+DEFAULT_BOOTSTRAP_SERVERS = "localhost:9092"
+DEFAULT_TOPIC = "product-sales-case"
+DEFAULT_GROUP_ID = "streaming-consumer-group-A"
+DEFAULT_AUTO_OFFSET_RESET = "earliest"
+
+
+# == DECLARE A FROZEN (IMMUTABLE) DATA CLASS FOR KAFKA SETTINGS ===
+
+
+@dataclass(frozen=True)
+class KafkaSettings:
+    """Kafka settings for producer and consumer examples."""
+
+    bootstrap_servers: str
+    topic: str
+    group_id: str
+    auto_offset_reset: str
+
+    @classmethod
+    def from_env(cls) -> Self:
+        """Create Kafka settings from environment variables."""
+        load_dotenv()
+
+        return cls(
+            bootstrap_servers=os.getenv(
+                "KAFKA_BOOTSTRAP_SERVERS",
+                DEFAULT_BOOTSTRAP_SERVERS,
+            ),
+            topic=os.getenv("KAFKA_TOPIC", DEFAULT_TOPIC),
+            group_id=os.getenv("KAFKA_GROUP_ID", DEFAULT_GROUP_ID),
+            auto_offset_reset=os.getenv(
+                "KAFKA_AUTO_OFFSET_RESET",
+                DEFAULT_AUTO_OFFSET_RESET,
+            ),
+        )
+
+    def producer_config(self) -> dict[str, str]:
+        """Return a confluent-kafka producer configuration."""
+        return {
+            "bootstrap.servers": self.bootstrap_servers,
+            "log_level": "3",
+            "message.timeout.ms": "5000",
+            "socket.timeout.ms": "5000",
+            "request.timeout.ms": "5000",
+        }
+
+    def consumer_config(self) -> dict[str, str]:
+        """Return a confluent-kafka consumer configuration."""
+        return {
+            "bootstrap.servers": self.bootstrap_servers,
+            "log_level": "3",
+            "group.id": self.group_id,
+            "auto.offset.reset": self.auto_offset_reset,
+        }

datafun_streaming/stats/__init__.py

@@ -0,0 +1 @@
+"""Statistical utilities for streaming data analysis."""

datafun_streaming/stats/stats_utils.py

@@ -0,0 +1,110 @@
+"""stats/stats_utils.py.
+
+Running statistics for streaming data.
+
+Provides a RunningStats class that tracks count, sum, mean, min, and max
+for a stream of numeric values without storing the full history.
+
+This is domain-agnostic: it works on any numeric field from any message.
+Pass it a value on each message and read the current statistics at any time.
+
+Author: Denise Case
+Date: 2026-05
+"""
+
+# === IMPORTS ===
+
+from dataclasses import dataclass
+
+# === EXPORTS ===
+
+__all__ = [
+    "RunningStats",
+]
+
+# === DEFINE RUNNING STATS CLASS ===
+
+
+@dataclass
+class RunningStats:
+    """Accumulates running statistics for a stream of numeric values.
+
+    Updates incrementally (one value at a time) without storing history.
+    Safe to use inside a message processing loop.
+
+    Do not use min and max as they would conflict with
+    built-in functions.
+    Access minimum and maximum values
+    via the minimum and maximum attributes.
+
+    Attributes:
+        count: Number of values received so far.
+        total: Running sum of all values.
+        mean:  Running mean of all values.
+        minimum:   Minimum value seen so far.
+        maximum:   Maximum value seen so far.
+
+    Example:
+        stats = RunningStats()
+        for message in messages:
+            stats.update(message["total"])
+            print(f"count={stats.count}  mean={stats.mean:.2f}")
+    """
+
+    count: int = 0
+    total: float = 0.0
+    mean: float = 0.0
+    minimum: float = float("inf")
+    maximum: float = float("-inf")
+
+    def update(self, value: float) -> None:
+        """Update statistics with one new value.
+
+        Arguments:
+            value: The new numeric value to include.
+
+        Returns:
+            None.
+        """
+        self.count += 1
+        self.total += value
+        self.mean = self.total / self.count
+        if value < self.minimum:
+            self.minimum = value
+        if value > self.maximum:
+            self.maximum = value
+
+    def reset(self) -> None:
+        """Reset all statistics to their initial state.
+
+        Use this to start a new window or clear accumulated state.
+
+        Returns:
+            None.
+        """
+        self.count = 0
+        self.total = 0.0
+        self.mean = 0.0
+        self.minimum = float("inf")
+        self.maximum = float("-inf")
+
+    @property
+    def is_empty(self) -> bool:
+        """Return True if no values have been received yet."""
+        return self.count == 0
+
+    def summary(self) -> str:
+        """Return a formatted summary string for logging.
+
+        Returns:
+            A single-line string with all current statistics.
+        """
+        if self.is_empty:
+            return "RunningStats: no values received yet."
+        return (
+            f"count={self.count}  "
+            f"total={self.total:,.2f}  "
+            f"mean={self.mean:,.2f}  "
+            f"minimum={self.minimum:,.2f}  "
+            f"maximum={self.maximum:,.2f}"
+        )

datafun_streaming/storage/__init__.py

@@ -0,0 +1 @@
+"""Persistence backends for streaming data."""