moose-lib 0.6.95__tar.gz → 0.6.97__tar.gz

{moose_lib-0.6.95 → moose_lib-0.6.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.6.95
+Version: 0.6.97
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com

{moose_lib-0.6.95 → moose_lib-0.6.97}/moose_lib/commons.py

@@ -4,8 +4,9 @@ from datetime import datetime, timezone
 
 import requests
 import json
-from typing import Optional, Literal
+from typing import Optional, Literal, Any, Union, Callable
 import os
+from kafka import KafkaConsumer, KafkaProducer
 
 
 class CliLogData:
@@ -100,3 +101,84 @@ class EnhancedJSONEncoder(json.JSONEncoder):
         if dataclasses.is_dataclass(o):
             return dataclasses.asdict(o)
         return super().default(o)
+
+
+def _build_kafka_kwargs(
+        broker: Union[str, list[str]],
+        sasl_username: Optional[str] = None,
+        sasl_password: Optional[str] = None,
+        sasl_mechanism: Optional[str] = None,
+        security_protocol: Optional[str] = None,
+) -> dict[str, Any]:
+    """Builds common Kafka client kwargs from provided parameters."""
+    kwargs: dict[str, Any] = {
+        "bootstrap_servers": broker,
+    }
+    if sasl_mechanism:
+        kwargs["sasl_mechanism"] = sasl_mechanism
+    if sasl_username is not None:
+        kwargs["sasl_plain_username"] = sasl_username
+    if sasl_password is not None:
+        kwargs["sasl_plain_password"] = sasl_password
+    if security_protocol is not None:
+        kwargs["security_protocol"] = security_protocol
+    return kwargs
+
+
+def get_kafka_consumer(
+        *,
+        broker: Union[str, list[str]],
+        client_id: str,
+        group_id: str,
+        sasl_username: Optional[str] = None,
+        sasl_password: Optional[str] = None,
+        sasl_mechanism: Optional[str] = None,
+        security_protocol: Optional[str] = None,
+        value_deserializer=lambda m: json.loads(m.decode("utf-8")),
+        **extra_kwargs: Any,
+) -> KafkaConsumer:
+    """Creates a configured KafkaConsumer with optional SASL/security settings."""
+    kwargs = _build_kafka_kwargs(
+        broker,
+        sasl_username=sasl_username,
+        sasl_password=sasl_password,
+        sasl_mechanism=sasl_mechanism,
+        security_protocol=security_protocol,
+    )
+    return KafkaConsumer(
+        client_id=client_id,
+        group_id=group_id,
+        value_deserializer=value_deserializer,
+        **kwargs,
+        **extra_kwargs,
+    )
+
+
+def get_kafka_producer(
+        *,
+        broker: Union[str, list[str]],
+        sasl_username: Optional[str] = None,
+        sasl_password: Optional[str] = None,
+        sasl_mechanism: Optional[str] = None,
+        security_protocol: Optional[str] = None,
+        max_request_size: Optional[int] = None,
+        value_serializer: Optional[Callable[[Any], bytes]] = None,
+        **extra_kwargs: Any,
+) -> KafkaProducer:
+    """Creates a configured KafkaProducer with optional SASL/security settings.
+    """
+    kwargs = _build_kafka_kwargs(
+        broker,
+        sasl_username=sasl_username,
+        sasl_password=sasl_password,
+        sasl_mechanism=sasl_mechanism,
+        security_protocol=security_protocol,
+    )
+    if max_request_size is not None:
+        kwargs["max_request_size"] = max_request_size
+    kwargs["max_in_flight_requests_per_connection"] = 1
+    if value_serializer is not None:
+        kwargs["value_serializer"] = value_serializer
+    # Allow callers to pass through additional Kafka configs like linger_ms, acks, retries, etc.
+    kwargs.update(extra_kwargs)
+    return KafkaProducer(**kwargs)

{moose_lib-0.6.95 → moose_lib-0.6.97}/moose_lib/config/config_file.py

@@ -9,6 +9,7 @@ import tomllib
 from dataclasses import dataclass
 from typing import Optional
 
+
 @dataclass
 class ClickHouseConfig:
     """ClickHouse configuration settings from moose.config.toml."""
@@ -20,11 +21,26 @@ class ClickHouseConfig:
     use_ssl: bool = False
     native_port: Optional[int] = None
 
+
+@dataclass
+class KafkaConfig:
+    """Redpanda/Kafka configuration settings from moose.config.toml."""
+    broker: str
+    message_timeout_ms: int
+    sasl_username: Optional[str] = None
+    sasl_password: Optional[str] = None
+    sasl_mechanism: Optional[str] = None
+    security_protocol: Optional[str] = None
+    namespace: Optional[str] = None
+
+
 @dataclass
 class ProjectConfig:
     """Project configuration from moose.config.toml."""
     language: str
     clickhouse_config: ClickHouseConfig
+    kafka_config: Optional[KafkaConfig] = None
+
 
 def find_config_file(start_dir: str = os.getcwd()) -> Optional[str]:
     """Find moose.config.toml by walking up directory tree.
@@ -48,6 +64,7 @@ def find_config_file(start_dir: str = os.getcwd()) -> Optional[str]:
         current_dir = parent_dir
     return None
 
+
 def read_project_config() -> ProjectConfig:
     """Read and parse moose.config.toml.
 
@@ -74,9 +91,28 @@ def read_project_config() -> ProjectConfig:
             native_port=config_data["clickhouse_config"].get("native_port")
         )
 
+        def _parse_kafka(section_name: str) -> Optional[KafkaConfig]:
+            sec = config_data.get(section_name)
+            if sec is None:
+                return None
+            return KafkaConfig(
+                broker=sec["broker"],
+                message_timeout_ms=sec.get("message_timeout_ms", 1000),
+                sasl_username=sec.get("sasl_username"),
+                sasl_password=sec.get("sasl_password"),
+                sasl_mechanism=sec.get("sasl_mechanism"),
+                security_protocol=sec.get("security_protocol"),
+                namespace=sec.get("namespace"),
+            )
+
+        kafka_cfg = _parse_kafka("kafka_config")
+        if kafka_cfg is None:
+            kafka_cfg = _parse_kafka("redpanda_config")
+
         return ProjectConfig(
             language=config_data["language"],
-            clickhouse_config=clickhouse_config
+            clickhouse_config=clickhouse_config,
+            kafka_config=kafka_cfg,
         )
     except Exception as e:
         raise RuntimeError(f"Failed to parse moose.config.toml: {e}")

moose_lib-0.6.97/moose_lib/config/runtime.py

@@ -0,0 +1,198 @@
+"""
+Runtime configuration management for Moose.
+
+This module provides a singleton registry for managing runtime configuration settings,
+particularly for ClickHouse connections.
+"""
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class RuntimeClickHouseConfig:
+    """Runtime ClickHouse configuration settings."""
+    host: str
+    port: str
+    username: str
+    password: str
+    database: str
+    use_ssl: bool
+
+
+@dataclass
+class RuntimeKafkaConfig:
+    """Runtime Kafka configuration settings."""
+    broker: str
+    message_timeout_ms: int
+    sasl_username: Optional[str]
+    sasl_password: Optional[str]
+    sasl_mechanism: Optional[str]
+    security_protocol: Optional[str]
+    namespace: Optional[str]
+
+
+class ConfigurationRegistry:
+    """Singleton registry for managing runtime configuration.
+
+    This class provides a centralized way to manage and access runtime configuration
+    settings, with fallback to file-based configuration when runtime settings are not set.
+    """
+    _instance: Optional['ConfigurationRegistry'] = None
+    _clickhouse_config: Optional[RuntimeClickHouseConfig] = None
+    _kafka_config: Optional[RuntimeKafkaConfig] = None
+
+    @classmethod
+    def get_instance(cls) -> 'ConfigurationRegistry':
+        """Get the singleton instance of ConfigurationRegistry.
+
+        Returns:
+            The singleton ConfigurationRegistry instance.
+        """
+        if not cls._instance:
+            cls._instance = cls()
+        return cls._instance
+
+    def set_clickhouse_config(self, config: RuntimeClickHouseConfig) -> None:
+        """Set the runtime ClickHouse configuration.
+
+        Args:
+            config: The ClickHouse configuration to use.
+        """
+        self._clickhouse_config = config
+
+    def set_kafka_config(self, config: 'RuntimeKafkaConfig') -> None:
+        """Set the runtime Kafka configuration.
+
+        Args:
+            config: The Kafka configuration to use.
+        """
+        self._kafka_config = config
+
+    def get_clickhouse_config(self) -> RuntimeClickHouseConfig:
+        """Get the current ClickHouse configuration.
+
+        If runtime configuration is not set, falls back to reading from moose.config.toml.
+
+        Returns:
+            The current ClickHouse configuration.
+        """
+        if self._clickhouse_config:
+            return self._clickhouse_config
+
+        # Fallback to reading from config file
+        from .config_file import read_project_config
+
+        def _env(name: str) -> Optional[str]:
+            val = os.environ.get(name)
+            if val is None:
+                return None
+            trimmed = val.strip()
+            return trimmed if trimmed else None
+
+        def _parse_bool(val: Optional[str]) -> Optional[bool]:
+            if val is None:
+                return None
+            v = val.strip().lower()
+            if v in ("1", "true", "yes", "on"):
+                return True
+            if v in ("0", "false", "no", "off"):
+                return False
+            return None
+
+        try:
+            config = read_project_config()
+
+            env_host = _env("MOOSE_CLICKHOUSE_CONFIG__HOST")
+            env_port = _env("MOOSE_CLICKHOUSE_CONFIG__HOST_PORT")
+            env_user = _env("MOOSE_CLICKHOUSE_CONFIG__USER")
+            env_password = _env("MOOSE_CLICKHOUSE_CONFIG__PASSWORD")
+            env_db = _env("MOOSE_CLICKHOUSE_CONFIG__DB_NAME")
+            env_use_ssl = _parse_bool(_env("MOOSE_CLICKHOUSE_CONFIG__USE_SSL"))
+
+            return RuntimeClickHouseConfig(
+                host=env_host or config.clickhouse_config.host,
+                port=(env_port or str(config.clickhouse_config.host_port)),
+                username=env_user or config.clickhouse_config.user,
+                password=env_password or config.clickhouse_config.password,
+                database=env_db or config.clickhouse_config.db_name,
+                use_ssl=(env_use_ssl if env_use_ssl is not None else config.clickhouse_config.use_ssl),
+            )
+        except Exception as e:
+            raise RuntimeError(f"Failed to get ClickHouse configuration: {e}")
+
+    def get_kafka_config(self) -> 'RuntimeKafkaConfig':
+        """Get the current Kafka configuration.
+
+        If runtime configuration is not set, falls back to reading from moose.config.toml
+        and environment variables (Redpanda- and Kafka-prefixed).
+
+        Returns:
+            The current Kafka configuration.
+        """
+        if self._kafka_config:
+            return self._kafka_config
+
+        from .config_file import read_project_config
+
+        def _env(name: str) -> Optional[str]:
+            val = os.environ.get(name)
+            if val is None:
+                return None
+            trimmed = val.strip()
+            return trimmed if trimmed else None
+
+        try:
+            config = read_project_config()
+
+            # Prefer Redpanda-prefixed env vars; fallback to Kafka-prefixed
+            broker = _env("MOOSE_REDPANDA_CONFIG__BROKER") or \
+                     _env("MOOSE_KAFKA_CONFIG__BROKER")
+            message_timeout_ms = _env("MOOSE_REDPANDA_CONFIG__MESSAGE_TIMEOUT_MS") or \
+                                 _env("MOOSE_KAFKA_CONFIG__MESSAGE_TIMEOUT_MS")
+            sasl_username = _env("MOOSE_REDPANDA_CONFIG__SASL_USERNAME") or \
+                            _env("MOOSE_KAFKA_CONFIG__SASL_USERNAME")
+            sasl_password = _env("MOOSE_REDPANDA_CONFIG__SASL_PASSWORD") or \
+                            _env("MOOSE_KAFKA_CONFIG__SASL_PASSWORD")
+            sasl_mechanism = _env("MOOSE_REDPANDA_CONFIG__SASL_MECHANISM") or \
+                             _env("MOOSE_KAFKA_CONFIG__SASL_MECHANISM")
+            security_protocol = _env("MOOSE_REDPANDA_CONFIG__SECURITY_PROTOCOL") or \
+                                _env("MOOSE_KAFKA_CONFIG__SECURITY_PROTOCOL")
+            namespace = _env("MOOSE_REDPANDA_CONFIG__NAMESPACE") or \
+                        _env("MOOSE_KAFKA_CONFIG__NAMESPACE")
+
+            file_kafka = config.kafka_config
+
+            def _to_int(value: Optional[str], fallback: int) -> int:
+                try:
+                    return int(value) if value is not None else fallback
+                except Exception:
+                    return fallback
+
+            return RuntimeKafkaConfig(
+                broker=broker or (file_kafka.broker if file_kafka else "localhost:19092"),
+                message_timeout_ms=_to_int(message_timeout_ms, file_kafka.message_timeout_ms if file_kafka else 1000),
+                sasl_username=sasl_username if sasl_username is not None else (
+                    file_kafka.sasl_username if file_kafka else None),
+                sasl_password=sasl_password if sasl_password is not None else (
+                    file_kafka.sasl_password if file_kafka else None),
+                sasl_mechanism=sasl_mechanism if sasl_mechanism is not None else (
+                    file_kafka.sasl_mechanism if file_kafka else None),
+                security_protocol=security_protocol if security_protocol is not None else (
+                    file_kafka.security_protocol if file_kafka else None),
+                namespace=namespace if namespace is not None else (file_kafka.namespace if file_kafka else None),
+            )
+        except Exception as e:
+            raise RuntimeError(f"Failed to get Kafka configuration: {e}")
+
+    def has_runtime_config(self) -> bool:
+        """Check if runtime configuration is set.
+
+        Returns:
+            True if either runtime clickhouse or kafka configuration is set, False otherwise.
+        """
+        return self._clickhouse_config is not None or self._kafka_config is not None
+
+
+# Create singleton instance
+config_registry = ConfigurationRegistry.get_instance()

{moose_lib-0.6.95 → moose_lib-0.6.97}/moose_lib/dmv2/ingest_pipeline.py

@@ -4,8 +4,9 @@ Ingestion Pipeline definitions for Moose Data Model v2 (dmv2).
 This module provides classes for defining and configuring complete ingestion pipelines,
 which combine tables, streams, and ingestion APIs into a single cohesive unit.
 """
+import warnings
 from typing import Any, Optional, Generic, TypeVar
-from pydantic import BaseModel
+from pydantic import BaseModel, model_validator
 
 from .types import TypedMooseResource, T
 from .olap_table import OlapTable, OlapConfig
@@ -23,7 +24,7 @@ class IngestPipelineConfig(BaseModel):
     Attributes:
         table: Configuration for the OLAP table component.
         stream: Configuration for the stream component.
-        ingest: Configuration for the ingest API component.
+        ingest_api: Configuration for the ingest API component.
         dead_letter_queue: Configuration for the dead letter queue.
         version: Optional version string applied to all created components.
         path: Optional custom path for the ingestion API endpoint.
@@ -32,12 +33,34 @@ class IngestPipelineConfig(BaseModel):
     """
     table: bool | OlapConfig = True
     stream: bool | StreamConfig = True
-    ingest: bool | IngestConfig = True
+    ingest_api: bool | IngestConfig = True
     dead_letter_queue: bool | StreamConfig = True
     version: Optional[str] = None
     path: Optional[str] = None
     metadata: Optional[dict] = None
     life_cycle: Optional[LifeCycle] = None
+    
+    # Legacy support - will be removed in future version
+    ingest: Optional[bool | IngestConfig] = None
+    
+    @model_validator(mode='before')
+    @classmethod
+    def handle_legacy_ingest_param(cls, data):
+        """Handle backwards compatibility for the deprecated 'ingest' parameter."""
+        if isinstance(data, dict) and 'ingest' in data:
+            warnings.warn(
+                "The 'ingest' parameter is deprecated and will be removed in a future version. "
+                "Please use 'ingest_api' instead.",
+                DeprecationWarning,
+                stacklevel=3
+            )
+            # If ingest_api is not explicitly set, use the ingest value
+            if 'ingest_api' not in data:
+                data['ingest_api'] = data['ingest']
+            # Remove the legacy parameter
+            data = data.copy()
+            del data['ingest']
+        return data
 
 class IngestPipeline(TypedMooseResource, Generic[T]):
     """Creates and configures a linked Table, Stream, and Ingest API pipeline.
@@ -150,11 +173,11 @@ class IngestPipeline(TypedMooseResource, Generic[T]):
                 stream_config.version = config.version
             stream_config.metadata = stream_metadata
             self.stream = Stream(name, stream_config, t=self._t)
-        if config.ingest:
+        if config.ingest_api:
             if self.stream is None:
                 raise ValueError("Ingest API needs a stream to write to.")
             ingest_config_dict = (
-                IngestConfig() if config.ingest is True else config.ingest
+                IngestConfig() if config.ingest_api is True else config.ingest_api
             ).model_dump()
             ingest_config_dict["destination"] = self.stream
             if config.version:

{moose_lib-0.6.95 → moose_lib-0.6.97}/moose_lib/dmv2/stream.py

@@ -6,14 +6,18 @@ including stream transformations, consumers, and dead letter queues.
 """
 import dataclasses
 import datetime
+import json
 from typing import Any, Optional, Callable, Union, Literal, Generic
 from pydantic import BaseModel, ConfigDict, AliasGenerator
 from pydantic.alias_generators import to_camel
+from kafka import KafkaProducer
 
 from .types import TypedMooseResource, ZeroOrMany, T, U
 from .olap_table import OlapTable
 from ._registry import _streams
 from .life_cycle import LifeCycle
+from ..config.runtime import config_registry, RuntimeKafkaConfig
+from ..commons import get_kafka_producer
 
 
 class StreamConfig(BaseModel):
@@ -110,6 +114,8 @@ class Stream(TypedMooseResource, Generic[T]):
     consumers: list[ConsumerEntry[T]]
     _multipleTransformations: Optional[Callable[[T], list[_RoutedMessage]]] = None
     default_dead_letter_queue: "Optional[DeadLetterQueue[T]]" = None
+    _memoized_producer: Optional[KafkaProducer] = None
+    _kafka_config_hash: Optional[str] = None
 
     def __init__(self, name: str, config: "StreamConfig" = None, **kwargs):
         super().__init__()
@@ -216,6 +222,116 @@ class Stream(TypedMooseResource, Generic[T]):
         """
         self._multipleTransformations = transformation
 
+    def _build_full_topic_name(self, namespace: Optional[str]) -> str:
+        """Build full topic name with optional namespace and version suffix."""
+        version_suffix = f"_{self.config.version.replace('.', '_')}" if self.config.version else ""
+        base = f"{self.name}{version_suffix}"
+        return f"{namespace}.{base}" if namespace else base
+
+    def _create_kafka_config_hash(self, cfg: RuntimeKafkaConfig) -> str:
+        import hashlib
+        config_string = ":".join(
+            str(x)
+            for x in (
+                cfg.broker,
+                cfg.message_timeout_ms,
+                cfg.sasl_username,
+                cfg.sasl_password,
+                cfg.sasl_mechanism,
+                cfg.security_protocol,
+                cfg.namespace,
+            )
+        )
+        return hashlib.sha256(config_string.encode()).hexdigest()[:16]
+
+    def _parse_brokers(self, broker_string: str) -> list[str]:
+        if not broker_string:
+            return []
+        return [b.strip() for b in broker_string.split(",") if b.strip()]
+
+    def _get_memoized_producer(self) -> tuple[KafkaProducer, Any]:
+        """Create or reuse a KafkaProducer using runtime configuration."""
+        cfg = config_registry.get_kafka_config()
+        current_hash = self._create_kafka_config_hash(cfg)
+
+        if self._memoized_producer is not None and self._kafka_config_hash == current_hash:
+            return self._memoized_producer, cfg
+
+        # Close previous producer if config changed
+        if self._memoized_producer is not None and self._kafka_config_hash != current_hash:
+            try:
+                self._memoized_producer.flush()
+                self._memoized_producer.close()
+            except Exception:
+                pass
+            finally:
+                self._memoized_producer = None
+
+        brokers = self._parse_brokers(getattr(cfg, "broker", "localhost:19092"))
+        if not brokers:
+            raise RuntimeError(f"No valid broker addresses found in: '{getattr(cfg, 'broker', '')}'")
+
+        producer = get_kafka_producer(
+            broker=brokers,
+            sasl_username=getattr(cfg, "sasl_username", None),
+            sasl_password=getattr(cfg, "sasl_password", None),
+            sasl_mechanism=getattr(cfg, "sasl_mechanism", None),
+            security_protocol=getattr(cfg, "security_protocol", None),
+            value_serializer=lambda v: v.model_dump_json().encode('utf-8'),
+            acks="all",
+        )
+
+        self._memoized_producer = producer
+        self._kafka_config_hash = current_hash
+        return producer, cfg
+
+    def close_producer(self) -> None:
+        """Closes the memoized Kafka producer if it exists."""
+        if self._memoized_producer is not None:
+            try:
+                self._memoized_producer.flush()
+                self._memoized_producer.close()
+            except Exception:
+                pass
+            finally:
+                self._memoized_producer = None
+                self._kafka_config_hash = None
+
+    def send(self, values: ZeroOrMany[T]) -> None:
+        """Send one or more records to this stream's Kafka topic.
+
+        Values are JSON-serialized using EnhancedJSONEncoder.
+        """
+        # Normalize inputs to a flat list of records
+        filtered: list[T] = []
+        if isinstance(values, list):
+            for v in values:
+                if v is None:
+                    continue
+                else:
+                    filtered.append(v)
+        elif values is not None:
+            filtered.append(values)  # type: ignore[arg-type]
+
+        if len(filtered) == 0:
+            return
+
+        # ensure all records are instances of the stream's model type
+        model_type = self._t
+        for rec in filtered:
+            if not isinstance(rec, model_type):
+                raise TypeError(
+                    f"Stream '{self.name}' expects instances of {model_type.__name__}, "
+                    f"got {type(rec).__name__}"
+                )
+
+        producer, cfg = self._get_memoized_producer()
+        topic = self._build_full_topic_name(getattr(cfg, "namespace", None))
+
+        for rec in filtered:
+            producer.send(topic, value=rec)
+        producer.flush()
+
 
 class DeadLetterModel(BaseModel, Generic[T]):
     """Model for dead letter queue messages.

{moose_lib-0.6.95 → moose_lib-0.6.97}/moose_lib/streaming/streaming_function_runner.py

@@ -30,7 +30,12 @@ from typing import Optional, Callable, Tuple, Any
 
 from moose_lib.dmv2 import get_streams, DeadLetterModel
 from moose_lib import cli_log, CliLogData, DeadLetterQueue
-from moose_lib.commons import EnhancedJSONEncoder, moose_management_port
+from moose_lib.commons import (
+    EnhancedJSONEncoder,
+    moose_management_port,
+    get_kafka_consumer,
+    get_kafka_producer,
+)
 
 # Force stdout to be unbuffered
 sys.stdout = io.TextIOWrapper(
@@ -294,29 +299,18 @@ def create_consumer() -> KafkaConsumer:
     Returns:
         Configured KafkaConsumer instance
     """
-    if sasl_config['mechanism'] is not None:
-        return KafkaConsumer(
-            source_topic.name,
-            client_id="python_streaming_function_consumer",
-            group_id=streaming_function_id,
-            bootstrap_servers=broker,
-            sasl_plain_username=sasl_config['username'],
-            sasl_plain_password=sasl_config['password'],
-            sasl_mechanism=sasl_config['mechanism'],
-            security_protocol=args.security_protocol,
-            # consumer_timeout_ms=10000,
-            value_deserializer=lambda m: json.loads(m.decode('utf-8'))
-        )
-    else:
-        log("No sasl mechanism specified. Using default consumer.")
-        return KafkaConsumer(
-            source_topic.name,
-            client_id="python_streaming_function_consumer",
-            group_id=streaming_function_id,
-            bootstrap_servers=broker,
-            # consumer_timeout_ms=10000,
-            value_deserializer=lambda m: json.loads(m.decode('utf-8'))
-        )
+    kwargs = dict(
+        broker=broker,
+        client_id="python_streaming_function_consumer",
+        group_id=streaming_function_id,
+        value_deserializer=lambda m: json.loads(m.decode("utf-8")),
+        sasl_username=sasl_config.get("username"),
+        sasl_password=sasl_config.get("password"),
+        sasl_mechanism=sasl_config.get("mechanism"),
+        security_protocol=args.security_protocol,
+    )
+    consumer = get_kafka_consumer(**kwargs)
+    return consumer
 
 
 def create_producer() -> Optional[KafkaProducer]:
@@ -330,20 +324,13 @@ def create_producer() -> Optional[KafkaProducer]:
     """
     max_request_size = KafkaProducer.DEFAULT_CONFIG['max_request_size'] if target_topic is None \
         else target_topic.max_message_bytes
-    if sasl_config['mechanism'] is not None:
-        return KafkaProducer(
-            bootstrap_servers=broker,
-            sasl_plain_username=sasl_config['username'],
-            sasl_plain_password=sasl_config['password'],
-            sasl_mechanism=sasl_config['mechanism'],
-            security_protocol=args.security_protocol,
-            max_request_size=max_request_size
-        )
-    log("No sasl mechanism specified. Using default producer.")
-    return KafkaProducer(
-        bootstrap_servers=broker,
-        max_in_flight_requests_per_connection=1,
-        max_request_size=max_request_size
+    return get_kafka_producer(
+        broker=broker,
+        sasl_username=sasl_config.get("username"),
+        sasl_password=sasl_config.get("password"),
+        sasl_mechanism=sasl_config.get("mechanism"),
+        security_protocol=args.security_protocol,
+        max_request_size=max_request_size,
     )
 
 
@@ -416,7 +403,6 @@ def main():
             kafka_refs['consumer'] = consumer
             kafka_refs['producer'] = producer
 
-            # Subscribe to topic
             consumer.subscribe([source_topic.name])
 
             log("Kafka consumer and producer initialized in processing thread")

{moose_lib-0.6.95 → moose_lib-0.6.97}/moose_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.6.95
+Version: 0.6.97
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com

moose_lib-0.6.95/moose_lib/config/runtime.py

@@ -1,110 +0,0 @@
-"""
-Runtime configuration management for Moose.
-
-This module provides a singleton registry for managing runtime configuration settings,
-particularly for ClickHouse connections.
-"""
-import os
-from dataclasses import dataclass
-from typing import Optional
-
-@dataclass
-class RuntimeClickHouseConfig:
-    """Runtime ClickHouse configuration settings."""
-    host: str
-    port: str
-    username: str
-    password: str
-    database: str
-    use_ssl: bool
-
-class ConfigurationRegistry:
-    """Singleton registry for managing runtime configuration.
-
-    This class provides a centralized way to manage and access runtime configuration
-    settings, with fallback to file-based configuration when runtime settings are not set.
-    """
-    _instance: Optional['ConfigurationRegistry'] = None
-    _clickhouse_config: Optional[RuntimeClickHouseConfig] = None
-
-    @classmethod
-    def get_instance(cls) -> 'ConfigurationRegistry':
-        """Get the singleton instance of ConfigurationRegistry.
-
-        Returns:
-            The singleton ConfigurationRegistry instance.
-        """
-        if not cls._instance:
-            cls._instance = cls()
-        return cls._instance
-
-    def set_clickhouse_config(self, config: RuntimeClickHouseConfig) -> None:
-        """Set the runtime ClickHouse configuration.
-
-        Args:
-            config: The ClickHouse configuration to use.
-        """
-        self._clickhouse_config = config
-
-    def get_clickhouse_config(self) -> RuntimeClickHouseConfig:
-        """Get the current ClickHouse configuration.
-
-        If runtime configuration is not set, falls back to reading from moose.config.toml.
-
-        Returns:
-            The current ClickHouse configuration.
-        """
-        if self._clickhouse_config:
-            return self._clickhouse_config
-
-        # Fallback to reading from config file
-        from .config_file import read_project_config
-
-        def _env(name: str) -> Optional[str]:
-            val = os.environ.get(name)
-            if val is None:
-                return None
-            trimmed = val.strip()
-            return trimmed if trimmed else None
-
-        def _parse_bool(val: Optional[str]) -> Optional[bool]:
-            if val is None:
-                return None
-            v = val.strip().lower()
-            if v in ("1", "true", "yes", "on"):
-                return True
-            if v in ("0", "false", "no", "off"):
-                return False
-            return None
-
-        try:
-            config = read_project_config()
-
-            env_host = _env("MOOSE_CLICKHOUSE_CONFIG__HOST")
-            env_port = _env("MOOSE_CLICKHOUSE_CONFIG__HOST_PORT")
-            env_user = _env("MOOSE_CLICKHOUSE_CONFIG__USER")
-            env_password = _env("MOOSE_CLICKHOUSE_CONFIG__PASSWORD")
-            env_db = _env("MOOSE_CLICKHOUSE_CONFIG__DB_NAME")
-            env_use_ssl = _parse_bool(_env("MOOSE_CLICKHOUSE_CONFIG__USE_SSL"))
-
-            return RuntimeClickHouseConfig(
-                host=env_host or config.clickhouse_config.host,
-                port=(env_port or str(config.clickhouse_config.host_port)),
-                username=env_user or config.clickhouse_config.user,
-                password=env_password or config.clickhouse_config.password,
-                database=env_db or config.clickhouse_config.db_name,
-                use_ssl=(env_use_ssl if env_use_ssl is not None else config.clickhouse_config.use_ssl),
-            )
-        except Exception as e:
-            raise RuntimeError(f"Failed to get ClickHouse configuration: {e}")
-
-    def has_runtime_config(self) -> bool:
-        """Check if runtime configuration is set.
-
-        Returns:
-            True if runtime configuration is set, False otherwise.
-        """
-        return self._clickhouse_config is not None
-
-# Create singleton instance
-config_registry = ConfigurationRegistry.get_instance()