moose-lib 0.6.63__tar.gz → 0.6.64__tar.gz

{moose_lib-0.6.63 → moose_lib-0.6.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.6.63
+Version: 0.6.64
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com

{moose_lib-0.6.63 → moose_lib-0.6.64}/moose_lib/dmv2/ingest_pipeline.py

@@ -132,22 +132,24 @@ class IngestPipeline(TypedMooseResource, Generic[T]):
                 table_config.version = config.version
             table_config.metadata = table_metadata
             self.table = OlapTable(name, table_config, t=self._t)
+        if config.dead_letter_queue:
+            dlq_stream_config = StreamConfig() if config.dead_letter_queue is True else config.dead_letter_queue
+            if config.version:
+                dlq_stream_config.version = config.version
+            dlq_stream_config.metadata = stream_metadata
+            self.dead_letter_queue = DeadLetterQueue(f"{name}DeadLetterQueue", dlq_stream_config, t=self._t)
         if config.stream:
             stream_config = (config.stream if isinstance(config.stream, StreamConfig) else 
                            StreamConfig(life_cycle=config.life_cycle))
             if config.table and stream_config.destination is not None:
                 raise ValueError("The destination of the stream should be the table created in the IngestPipeline")
             stream_config.destination = self.table
+            if self.dead_letter_queue is not None:
+                stream_config.default_dead_letter_queue = self.dead_letter_queue
             if config.version:
                 stream_config.version = config.version
             stream_config.metadata = stream_metadata
             self.stream = Stream(name, stream_config, t=self._t)
-        if config.dead_letter_queue:
-            stream_config = StreamConfig() if config.dead_letter_queue is True else config.dead_letter_queue
-            if config.version:
-                stream_config.version = config.version
-            stream_config.metadata = stream_metadata
-            self.dead_letter_queue = DeadLetterQueue(f"{name}DeadLetterQueue", stream_config, t=self._t)
         if config.ingest:
             if self.stream is None:
                 raise ValueError("Ingest API needs a stream to write to.")

{moose_lib-0.6.63 → moose_lib-0.6.64}/moose_lib/dmv2/stream.py

@@ -15,6 +15,7 @@ from .olap_table import OlapTable
 from ._registry import _streams
 from .life_cycle import LifeCycle
 
+
 class StreamConfig(BaseModel):
     """Configuration for data streams (e.g., Redpanda topics).
 
@@ -25,6 +26,7 @@ class StreamConfig(BaseModel):
         version: Optional version string for tracking configuration changes.
         metadata: Optional metadata for the stream.
         life_cycle: Determines how changes in code will propagate to the resources.
+        default_dead_letter_queue: default dead letter queue used by transforms/consumers
     """
     parallelism: int = 1
     retention_period: int = 60 * 60 * 24 * 7  # 7 days
@@ -32,6 +34,10 @@ class StreamConfig(BaseModel):
     version: Optional[str] = None
     metadata: Optional[dict] = None
     life_cycle: Optional[LifeCycle] = None
+    default_dead_letter_queue: "Optional[DeadLetterQueue]" = None
+    # allow DeadLetterQueue
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
 
 class TransformConfig(BaseModel):
     """Configuration for stream transformations.
@@ -45,6 +51,7 @@ class TransformConfig(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
     metadata: Optional[dict] = None
 
+
 class ConsumerConfig(BaseModel):
     """Configuration for stream consumers.
 
@@ -56,18 +63,21 @@ class ConsumerConfig(BaseModel):
     dead_letter_queue: "Optional[DeadLetterQueue]" = None
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
+
 @dataclasses.dataclass
 class _RoutedMessage:
     """Internal class representing a message routed to a specific stream."""
     destination: "Stream[Any]"
     values: ZeroOrMany[Any]
 
+
 @dataclasses.dataclass
 class ConsumerEntry(Generic[T]):
     """Internal class representing a consumer with its configuration."""
     consumer: Callable[[T], None]
     config: ConsumerConfig
 
+
 @dataclasses.dataclass
 class TransformEntry(Generic[T]):
     """Internal class representing a transformation with its configuration."""
@@ -75,6 +85,7 @@ class TransformEntry(Generic[T]):
     transformation: Callable[[T], ZeroOrMany[Any]]
     config: TransformConfig
 
+
 class Stream(TypedMooseResource, Generic[T]):
     """Represents a data stream (e.g., a Redpanda topic) typed with a Pydantic model.
 
@@ -98,14 +109,16 @@ class Stream(TypedMooseResource, Generic[T]):
     transformations: dict[str, list[TransformEntry[T]]]
     consumers: list[ConsumerEntry[T]]
     _multipleTransformations: Optional[Callable[[T], list[_RoutedMessage]]] = None
+    default_dead_letter_queue: "Optional[DeadLetterQueue[T]]" = None
 
-    def __init__(self, name: str, config: StreamConfig = StreamConfig(), **kwargs):
+    def __init__(self, name: str, config: "StreamConfig" = None, **kwargs):
         super().__init__()
         self._set_type(name, self._get_type(kwargs))
-        self.config = config
-        self.metadata = config.metadata
+        self.config = config if config is not None else StreamConfig()
+        self.metadata = self.config.metadata
         self.consumers = []
         self.transformations = {}
+        self.default_dead_letter_queue = self.config.default_dead_letter_queue
         _streams[name] = self
 
     def add_transform(self, destination: "Stream[U]", transformation: Callable[[T], ZeroOrMany[U]],
@@ -121,6 +134,12 @@ class Stream(TypedMooseResource, Generic[T]):
             config: Optional configuration, primarily for setting a version.
         """
         config = config or TransformConfig()
+        if (
+                self.default_dead_letter_queue is not None
+                and config.dead_letter_queue is None
+        ):
+            config = config.model_copy()
+            config.dead_letter_queue = self.default_dead_letter_queue
         if destination.name in self.transformations:
             existing_transforms = self.transformations[destination.name]
             # Check if a transform with this version already exists
@@ -142,6 +161,12 @@ class Stream(TypedMooseResource, Generic[T]):
             config: Optional configuration, primarily for setting a version.
         """
         config = config or ConsumerConfig()
+        if (
+                self.default_dead_letter_queue is not None
+                and config.dead_letter_queue is None
+        ):
+            config = config.model_copy()
+            config.dead_letter_queue = self.default_dead_letter_queue
         has_version = any(c.config.version == config.version for c in self.consumers)
         if not has_version:
             self.consumers.append(ConsumerEntry(consumer=consumer, config=config))
@@ -191,6 +216,7 @@ class Stream(TypedMooseResource, Generic[T]):
         """
         self._multipleTransformations = transformation
 
+
 class DeadLetterModel(BaseModel, Generic[T]):
     """Model for dead letter queue messages.
 
@@ -213,6 +239,7 @@ class DeadLetterModel(BaseModel, Generic[T]):
     def as_typed(self) -> T:
         return self._t.model_validate(self.original_record)
 
+
 class DeadLetterQueue(Stream, Generic[T]):
     """A specialized Stream for handling failed records.
 
@@ -225,7 +252,7 @@ class DeadLetterQueue(Stream, Generic[T]):
 
     _model_type: type[T]
 
-    def __init__(self, name: str, config: StreamConfig = StreamConfig(), **kwargs):
+    def __init__(self, name: str, config: "StreamConfig" = None, **kwargs):
         """Initialize a new DeadLetterQueue.
 
         Args:
@@ -234,7 +261,7 @@ class DeadLetterQueue(Stream, Generic[T]):
         """
         self._model_type = self._get_type(kwargs)
         kwargs["t"] = DeadLetterModel[self._model_type]
-        super().__init__(name, config, **kwargs)
+        super().__init__(name, config if config is not None else StreamConfig(), **kwargs)
 
     def add_transform(self, destination: Stream[U], transformation: Callable[[DeadLetterModel[T]], ZeroOrMany[U]],
                       config: TransformConfig = None):
@@ -258,4 +285,4 @@ class DeadLetterQueue(Stream, Generic[T]):
             record._t = self._model_type
             return transformation(record)
 
-        super().set_multi_transform(wrapped_transform)
+        super().set_multi_transform(wrapped_transform)

{moose_lib-0.6.63 → moose_lib-0.6.64}/moose_lib/streaming/streaming_function_runner.py

@@ -328,6 +328,8 @@ def create_producer() -> Optional[KafkaProducer]:
     Returns:
         Configured KafkaProducer instance
     """
+    max_request_size = KafkaProducer.DEFAULT_CONFIG['max_request_size'] if target_topic is None \
+        else target_topic.max_message_bytes
     if sasl_config['mechanism'] is not None:
         return KafkaProducer(
             bootstrap_servers=broker,
@@ -335,13 +337,13 @@ def create_producer() -> Optional[KafkaProducer]:
             sasl_plain_password=sasl_config['password'],
             sasl_mechanism=sasl_config['mechanism'],
             security_protocol=args.security_protocol,
-            max_request_size=target_topic.max_message_bytes
+            max_request_size=max_request_size
         )
     log("No sasl mechanism specified. Using default producer.")
     return KafkaProducer(
         bootstrap_servers=broker,
         max_in_flight_requests_per_connection=1,
-        max_request_size=target_topic.max_message_bytes
+        max_request_size=max_request_size
     )
 
 

{moose_lib-0.6.63 → moose_lib-0.6.64}/moose_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.6.63
+Version: 0.6.64
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com