moose-lib 0.6.110__tar.gz → 0.6.111__tar.gz

{moose_lib-0.6.110 → moose_lib-0.6.111}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.6.110
+Version: 0.6.111
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com
@@ -15,6 +15,7 @@ Requires-Dist: humanfriendly>=10.0
 Requires-Dist: clickhouse_connect>=0.7.16
 Requires-Dist: requests>=2.32.3
 Requires-Dist: sqlglot[rs]>=27.16.3
+Requires-Dist: confluent-kafka[json,schemaregistry]>=2.11.1
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{moose_lib-0.6.110 → moose_lib-0.6.111}/moose_lib/config/config_file.py

@@ -32,6 +32,7 @@ class KafkaConfig:
     sasl_mechanism: Optional[str] = None
     security_protocol: Optional[str] = None
     namespace: Optional[str] = None
+    schema_registry_url: Optional[str] = None
 
 
 @dataclass
@@ -103,6 +104,7 @@ def read_project_config() -> ProjectConfig:
                 sasl_mechanism=sec.get("sasl_mechanism"),
                 security_protocol=sec.get("security_protocol"),
                 namespace=sec.get("namespace"),
+                schema_registry_url=sec.get("schema_registry_url"),
             )
 
         kafka_cfg = _parse_kafka("kafka_config")

{moose_lib-0.6.110 → moose_lib-0.6.111}/moose_lib/config/runtime.py

@@ -30,6 +30,7 @@ class RuntimeKafkaConfig:
     sasl_mechanism: Optional[str]
     security_protocol: Optional[str]
     namespace: Optional[str]
+    schema_registry_url: Optional[str]
 
 
 class ConfigurationRegistry:
@@ -160,6 +161,8 @@ class ConfigurationRegistry:
                                 _env("MOOSE_KAFKA_CONFIG__SECURITY_PROTOCOL")
             namespace = _env("MOOSE_REDPANDA_CONFIG__NAMESPACE") or \
                         _env("MOOSE_KAFKA_CONFIG__NAMESPACE")
+            schema_registry_url = _env("MOOSE_REDPANDA_CONFIG__SCHEMA_REGISTRY_URL") or \
+                                  _env("MOOSE_KAFKA_CONFIG__SCHEMA_REGISTRY_URL")
 
             file_kafka = config.kafka_config
 
@@ -181,6 +184,8 @@ class ConfigurationRegistry:
                 security_protocol=security_protocol if security_protocol is not None else (
                     file_kafka.security_protocol if file_kafka else None),
                 namespace=namespace if namespace is not None else (file_kafka.namespace if file_kafka else None),
+                schema_registry_url=schema_registry_url if schema_registry_url is not None else (
+                    file_kafka.schema_registry_url if file_kafka else None),
             )
         except Exception as e:
             raise RuntimeError(f"Failed to get Kafka configuration: {e}")

{moose_lib-0.6.110 → moose_lib-0.6.111}/moose_lib/dmv2/__init__.py

@@ -28,6 +28,10 @@ from .stream import (
     Stream,
     DeadLetterModel,
     DeadLetterQueue,
+    SubjectLatest,
+    SubjectVersion,
+    SchemaById,
+    KafkaSchemaConfig,
 )
 
 from .ingest_api import (
@@ -117,6 +121,10 @@ __all__ = [
     'Stream',
     'DeadLetterModel',
     'DeadLetterQueue',
+    'SubjectLatest',
+    'SubjectVersion',
+    'SchemaById',
+    'KafkaSchemaConfig',
 
     # Ingestion
     'IngestConfig',

{moose_lib-0.6.110 → moose_lib-0.6.111}/moose_lib/dmv2/stream.py

@@ -8,7 +8,7 @@ import dataclasses
 import datetime
 import json
 from typing import Any, Optional, Callable, Union, Literal, Generic
-from pydantic import BaseModel, ConfigDict, AliasGenerator
+from pydantic import BaseModel, ConfigDict, AliasGenerator, Field
 from pydantic.alias_generators import to_camel
 from kafka import KafkaProducer
 
@@ -20,6 +20,24 @@ from ..config.runtime import config_registry, RuntimeKafkaConfig
 from ..commons import get_kafka_producer
 
 
+class SubjectLatest(BaseModel):
+    name: str = Field(serialization_alias="subjectLatest")
+
+
+class SubjectVersion(BaseModel):
+    subject: str
+    version: int
+
+
+class SchemaById(BaseModel):
+    id: int
+
+
+class KafkaSchemaConfig(BaseModel):
+    kind: Literal["JSON", "AVRO", "PROTOBUF"]
+    reference: Union[SubjectLatest, SubjectVersion, SchemaById]
+
+
 class StreamConfig(BaseModel):
     """Configuration for data streams (e.g., Redpanda topics).
 
@@ -41,6 +59,7 @@ class StreamConfig(BaseModel):
     default_dead_letter_queue: "Optional[DeadLetterQueue]" = None
     # allow DeadLetterQueue
     model_config = ConfigDict(arbitrary_types_allowed=True)
+    schema_config: Optional[KafkaSchemaConfig] = None
 
 
 class TransformConfig(BaseModel):
@@ -249,9 +268,9 @@ class Stream(TypedMooseResource, Generic[T]):
             return []
         return [b.strip() for b in broker_string.split(",") if b.strip()]
 
-    def _get_memoized_producer(self) -> tuple[KafkaProducer, Any]:
+    def _get_memoized_producer(self) -> tuple[KafkaProducer, RuntimeKafkaConfig]:
         """Create or reuse a KafkaProducer using runtime configuration."""
-        cfg = config_registry.get_kafka_config()
+        cfg: RuntimeKafkaConfig = config_registry.get_kafka_config()
         current_hash = self._create_kafka_config_hash(cfg)
 
         if self._memoized_producer is not None and self._kafka_config_hash == current_hash:
@@ -267,16 +286,16 @@ class Stream(TypedMooseResource, Generic[T]):
             finally:
                 self._memoized_producer = None
 
-        brokers = self._parse_brokers(getattr(cfg, "broker", "localhost:19092"))
+        brokers = self._parse_brokers(cfg.broker)
         if not brokers:
-            raise RuntimeError(f"No valid broker addresses found in: '{getattr(cfg, 'broker', '')}'")
+            raise RuntimeError(f"No valid broker addresses found in: '{cfg.broker}'")
 
         producer = get_kafka_producer(
             broker=brokers,
-            sasl_username=getattr(cfg, "sasl_username", None),
-            sasl_password=getattr(cfg, "sasl_password", None),
-            sasl_mechanism=getattr(cfg, "sasl_mechanism", None),
-            security_protocol=getattr(cfg, "security_protocol", None),
+            sasl_username=cfg.sasl_username,
+            sasl_password=cfg.sasl_password,
+            sasl_mechanism=cfg.sasl_mechanism,
+            security_protocol=cfg.security_protocol,
             value_serializer=lambda v: v.model_dump_json().encode('utf-8'),
             acks="all",
         )
@@ -300,7 +319,9 @@ class Stream(TypedMooseResource, Generic[T]):
     def send(self, values: ZeroOrMany[T]) -> None:
         """Send one or more records to this stream's Kafka topic.
 
-        Values are JSON-serialized using EnhancedJSONEncoder.
+        If `schema_registry` (JSON) is configured, resolve schema id and
+        send using Confluent wire format (0x00 + 4-byte schema id + JSON bytes).
+        Otherwise, values are JSON-serialized.
         """
         # Normalize inputs to a flat list of records
         filtered: list[T] = []
@@ -326,11 +347,42 @@ class Stream(TypedMooseResource, Generic[T]):
                 )
 
         producer, cfg = self._get_memoized_producer()
-        topic = self._build_full_topic_name(getattr(cfg, "namespace", None))
+        topic = self._build_full_topic_name(cfg.namespace)
 
-        for rec in filtered:
-            producer.send(topic, value=rec)
-        producer.flush()
+        sr = self.config.schema_config
+        if sr is not None:
+            if sr.kind != "JSON":
+                raise NotImplementedError("Currently JSON Schema is supported.")
+            try:
+                from confluent_kafka.schema_registry import SchemaRegistryClient
+                from confluent_kafka.schema_registry.json_schema import JSONSerializer
+            except Exception as e:
+                raise RuntimeError(
+                    "confluent-kafka[json,schemaregistry] is required for Schema Registry JSON"
+                ) from e
+
+            sr_url = cfg.schema_registry_url
+            if not sr_url:
+                raise RuntimeError("Schema Registry URL not configured")
+            client = SchemaRegistryClient({"url": sr_url})
+
+            if isinstance(sr.reference, SchemaById):
+                schema = client.get_schema(sr.reference.id)
+            elif isinstance(sr.reference, SubjectLatest):
+                schema = client.get_latest_version(sr.reference.name).schema
+            else:
+                schema = client.get_version(sr.reference.subject, sr.reference.version).schema
+
+            serializer = JSONSerializer(schema, client)
+
+            for rec in filtered:
+                value_bytes = serializer(rec.model_dump())
+                producer.send(topic, value=value_bytes)
+            producer.flush()
+        else:
+            for rec in filtered:
+                producer.send(topic, value=rec)
+            producer.flush()
 
 
 class DeadLetterModel(BaseModel, Generic[T]):

{moose_lib-0.6.110 → moose_lib-0.6.111}/moose_lib/internal.py

@@ -24,6 +24,7 @@ from moose_lib.dmv2 import (
     MaterializedView,
     SqlResource
 )
+from moose_lib.dmv2.stream import KafkaSchemaConfig
 from pydantic.alias_generators import to_camel
 from pydantic.json_schema import JsonSchemaValue
 
@@ -163,6 +164,7 @@ class TopicConfig(BaseModel):
     consumers: List[Consumer]
     metadata: Optional[dict] = None
     life_cycle: Optional[str] = None
+    schema_config: Optional[KafkaSchemaConfig] = None
 
 
 class IngestApiConfig(BaseModel):
@@ -477,6 +479,7 @@ def to_infra_map() -> dict:
             consumers=consumers,
             metadata=getattr(stream, "metadata", None),
             life_cycle=stream.config.life_cycle.value if stream.config.life_cycle else None,
+            schema_config=stream.config.schema_config,
         )
 
     for name, api in get_ingest_apis().items():

{moose_lib-0.6.110 → moose_lib-0.6.111}/moose_lib/streaming/streaming_function_runner.py

@@ -299,11 +299,19 @@ def create_consumer() -> KafkaConsumer:
     Returns:
         Configured KafkaConsumer instance
     """
+    def _sr_json_deserializer(m: bytes):
+        if m is None:
+            return None
+        # Schema Registry JSON envelope: 0x00 + 4-byte schema ID (big-endian) + JSON
+        if len(m) >= 5 and m[0] == 0x00:
+            m = m[5:]
+        return json.loads(m.decode("utf-8"))
+
     kwargs = dict(
         broker=broker,
         client_id="python_streaming_function_consumer",
         group_id=streaming_function_id,
-        value_deserializer=lambda m: json.loads(m.decode("utf-8")),
+        value_deserializer=_sr_json_deserializer,
         sasl_username=sasl_config.get("username"),
         sasl_password=sasl_config.get("password"),
         sasl_mechanism=sasl_config.get("mechanism"),

{moose_lib-0.6.110 → moose_lib-0.6.111}/moose_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.6.110
+Version: 0.6.111
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com
@@ -15,6 +15,7 @@ Requires-Dist: humanfriendly>=10.0
 Requires-Dist: clickhouse_connect>=0.7.16
 Requires-Dist: requests>=2.32.3
 Requires-Dist: sqlglot[rs]>=27.16.3
+Requires-Dist: confluent-kafka[json,schemaregistry]>=2.11.1
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{moose_lib-0.6.110 → moose_lib-0.6.111}/moose_lib.egg-info/requires.txt

@@ -7,3 +7,4 @@ humanfriendly>=10.0
 clickhouse_connect>=0.7.16
 requests>=2.32.3
 sqlglot[rs]>=27.16.3
+confluent-kafka[json,schemaregistry]>=2.11.1

{moose_lib-0.6.110 → moose_lib-0.6.111}/setup.py

@@ -33,5 +33,6 @@ setup(
         "clickhouse_connect>=0.7.16",
         "requests>=2.32.3",
         "sqlglot[rs]>=27.16.3",
+        "confluent-kafka[json,schemaregistry]>=2.11.1"
     ],
 )