dgkafka 1.0.0a7__py3-none-any.whl → 1.0.0a9__py3-none-any.whl

dgkafka/avro_consumer.py

@@ -38,7 +38,7 @@ class AvroKafkaConsumer(KafkaConsumer):
             self.logger.error(f"[x] Failed to initialize Avro consumer: {ex}")
             raise
 
-    def consume(self, num_messages: int = 1, timeout: float = 10.0, **kwargs: Any) -> Iterator[str | bytes | Message | None]:
+    def consume(self, num_messages: int = 1, timeout: float = 1.0, **kwargs: Any) -> Iterator[str | bytes | Message | None]:
         """
         Consume Avro-encoded messages.
 

dgkafka/config.py

@@ -1,92 +1,82 @@
-from typing import Literal, Optional
-from pydantic import BaseModel, Field, validator, Json
+from typing import Dict, Any, Optional, Literal
+from pydantic import BaseModel, Field, ConfigDict
 from enum import Enum
 
-
 class SecurityProtocol(str, Enum):
     PLAINTEXT = "PLAINTEXT"
     SSL = "SSL"
     SASL_PLAINTEXT = "SASL_PLAINTEXT"
     SASL_SSL = "SASL_SSL"
 
-
 class KafkaConfig(BaseModel):
     """Base configuration for all Kafka clients"""
-    bootstrap_servers: str = Field(..., alias="bootstrap.servers",
-                                   description="Comma-separated list of broker addresses")
-    security_protocol: SecurityProtocol = Field(default=SecurityProtocol.SSL,
+    bootstrap_servers: str = Field(..., alias="bootstrap.servers")
+    security_protocol: Literal["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"] = Field(default="SSL",
                                                 alias="security.protocol")
-    ssl_cafile: Optional[str] = Field(default=None, alias="ssl.ca.location")
-    ssl_certfile: Optional[str] = Field(default=None, alias="ssl.certificate.location")
-    ssl_keyfile: Optional[str] = Field(default=None, alias="ssl.key.location")
-    ssl_check_hostname: bool = Field(default=True, alias="enable.ssl.certificate.verification")
-    ssl_password: Optional[str] = Field(default=None, alias="ssl.key.password")
-
-    def get(self) -> dict[str, Json]:
-        return self.model_dump(by_alias=True)
-
-    class Config:
-        validate_by_name = True
-        use_enum_values = True
-        extra = "forbid"
+    ssl_ca_location: Optional[str] = Field(default=None, alias="ssl.ca.location")
+    ssl_certificate_location: Optional[str] = Field(default=None, alias="ssl.certificate.location")
+    ssl_key_location: Optional[str] = Field(default=None, alias="ssl.key.location")
+    ssl_endpoint_identification_algorithm: Optional[str] = Field(default=None,
+                                                                 alias="ssl.endpoint.identification.algorithm")
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        extra="forbid",
+        protected_namespaces=()
+    )
+
+    def get(self) -> Dict[str, Any]:
+        """Get config in format suitable for confluent_kafka"""
+        return self.model_dump(by_alias=True, exclude_none=True)
 
+    @classmethod
+    def set(cls, config_dict: Dict[str, Any]) -> "KafkaConfig":
+        """Create config from dictionary"""
+        return cls(**config_dict)
 
 class ConsumerConfig(KafkaConfig):
-    """Configuration for basic Kafka consumer"""
+    """Base consumer configuration"""
     group_id: str = Field(..., alias="group.id")
     enable_auto_commit: bool = Field(default=False, alias="enable.auto.commit")
-    auto_offset_reset: Literal["earliest", "latest"] = Field(default="latest",
-                                                             alias="auto.offset.reset")
+    auto_offset_reset: Literal["earliest", "latest"] = Field(
+        default="earliest", alias="auto.offset.reset")
     session_timeout_ms: int = Field(default=10000, alias="session.timeout.ms")
     max_poll_interval_ms: int = Field(default=300000, alias="max.poll.interval.ms")
-    fetch_max_bytes: int = Field(default=52428800, alias="fetch.max.bytes")
-
 
 class ProducerConfig(KafkaConfig):
-    """Configuration for basic Kafka producer"""
+    """Base producer configuration"""
     acks: Literal["all", "0", "1"] = Field(default="all")
     retries: int = Field(default=0)
-    linger_ms: int = Field(default=0, alias="linger.ms")
-    compression_type: Literal["none", "gzip", "snappy", "lz4", "zstd"] = Field(
-        default="none", alias="compression.type")
+    compression_type: str = Field(default="none", alias="compression.type")
     batch_size: int = Field(default=16384, alias="batch.size")
-    max_in_flight: int = Field(default=1000000, alias="max.in.flight.requests.per.connection")
-
-
-class AvroConsumerConfig(ConsumerConfig):
-    """Configuration for Avro consumer with Schema Registry"""
-    schema_registry_url: str = Field(..., alias="schema.registry.url")
-    schema_registry_ssl_cafile: Optional[str] = Field(default=None,
-                                                      alias="schema.registry.ssl.ca.location")
-    schema_registry_basic_auth_user_info: Optional[str] = Field(default=None,
-                                                                alias="schema.registry.basic.auth.user.info")
-    specific_avro_reader: bool = Field(default=False, alias="specific.avro.reader")
+    linger_ms: int = Field(default=0, alias="linger.ms")
 
 
-class AvroProducerConfig(ProducerConfig):
-    """Configuration for Avro producer with Schema Registry"""
+class AvroConfigMixin:
     schema_registry_url: str = Field(..., alias="schema.registry.url")
-    schema_registry_ssl_cafile: Optional[str] = Field(default=None,
-                                                      alias="schema.registry.ssl.ca.location")
-    schema_registry_basic_auth_user_info: Optional[str] = Field(default=None,
-                                                                alias="schema.registry.basic.auth.user.info")
-    max_schemas_per_subject: int = Field(default=1000, alias="max.schemas.per.subject")
+    schema_registry_ssl_ca_location: Optional[str] = Field(
+        default=None, alias="schema.registry.ssl.ca.location")
+    schema_registry_basic_auth_user_info: Optional[str] = Field(
+        default=None, alias="basic.auth.user.info")
 
 
-class JsonConsumerConfig(ConsumerConfig):
-    """Configuration for JSON consumer"""
-    json_deserializer: Optional[str] = None  # Custom deserializer function
-    encoding: str = Field(default="utf-8")
+class AvroConsumerConfig(ConsumerConfig, AvroConfigMixin):
+    """Avro consumer configuration with Schema Registry support"""
 
-
-class JsonProducerConfig(ProducerConfig):
-    """Configuration for JSON producer"""
-    json_serializer: Optional[str] = None  # Custom serializer function
-    encoding: str = Field(default="utf-8")
+    @classmethod
+    def set(cls, config_dict: Dict[str, Any]) -> "AvroConsumerConfig":
+        """Create from dictionary with Schema Registry validation"""
+        if "schema.registry.url" not in config_dict:
+            raise ValueError("schema.registry.url is required for AvroConsumer")
+        return cls(**config_dict)
+
+class AvroProducerConfig(ProducerConfig, AvroConfigMixin):
+    """Avro producer configuration with Schema Registry support"""
+    max_schemas_per_subject: int = Field(default=1000, alias="max.schemas.per.subject")
 
     @classmethod
-    @validator("compression_type")
-    def validate_compression(cls, v):
-        if v not in ["none", "gzip", "snappy", "lz4", "zstd"]:
-            raise ValueError("Unsupported compression type")
-        return v
+    def set(cls, config_dict: Dict[str, Any]) -> "AvroProducerConfig":
+        """Create from dictionary with Schema Registry validation"""
+        if "schema.registry.url" not in config_dict:
+            raise ValueError("schema.registry.url is required for AvroProducer")
+        return cls(**config_dict)

dgkafka/consumer.py

@@ -76,7 +76,7 @@ class KafkaConsumer:
         consumer.seek(topic_partition)
         self.logger.info(f"[*] Assigned to topic '{topic}' partition {partition} with offset {offset}")
 
-    def consume(self, num_messages: int = 1, timeout: float = 10.0, decode_utf8: bool = False) -> Iterator[Message | str]:
+    def consume(self, num_messages: int = 1, timeout: float = 1.0, decode_utf8: bool = False) -> Iterator[Message | str]:
         """Consume messages"""
         consumer = self._ensure_consumer()
 

{dgkafka-1.0.0a7.dist-info → dgkafka-1.0.0a9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dgkafka
-Version: 1.0.0a7
+Version: 1.0.0a9
 Summary: Kafka clients
 Home-page: https://gitlab.com/gng-group/dgkafka.git
 Author: Malanris

dgkafka-1.0.0a9.dist-info/RECORD

@@ -0,0 +1,13 @@
+dgkafka/__init__.py,sha256=fnqVZROyHXipdmhqZaa9XUjvQe795JJKFakwTndAiIw,286
+dgkafka/avro_consumer.py,sha256=By2QWA2BX6ih40x2w3nI1_bv4XtghoVeLcSBNcpJByw,2874
+dgkafka/avro_producer.py,sha256=51p9GeavU3M21LfQZuidEkhIpzA-KGQOXhB0xMfNUus,4247
+dgkafka/config.py,sha256=uaqllIJZS98_LXg7HE8KFBySZeLJ7BWw2Vv3aft0rtA,3679
+dgkafka/consumer.py,sha256=JoaNnAbCbhSjRl4X4ikReC5Ey5XamJS2DRPyYnP_RRM,4426
+dgkafka/errors.py,sha256=PaH46tXameS--hrrUXKhQkZlBHvMSMPmjhVeRkmFvV0,95
+dgkafka/json_consumer.py,sha256=7Gzn7C9WpyCTPDV6eFDugAx5gC9vdV-HrTh3Nv--zIE,1152
+dgkafka/producer.py,sha256=4tKIYUs1zlrie5TOxnL0J38BON2HreOLf2rYW1hVrIs,5085
+dgkafka-1.0.0a9.dist-info/licenses/LICENSE,sha256=pAZXnNE2dxxwXFIduGyn1gpvPefJtUYOYZOi3yeGG94,1068
+dgkafka-1.0.0a9.dist-info/METADATA,sha256=oBHVvRoOCDkFENPqaV7aK9LYnpgXKT5O_aP7XQFehYI,6061
+dgkafka-1.0.0a9.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dgkafka-1.0.0a9.dist-info/top_level.txt,sha256=GyNrxOh7IPdL0t2SxH8DWxg3fUma-ezQ1Kz4zIr2B7U,8
+dgkafka-1.0.0a9.dist-info/RECORD,,

dgkafka-1.0.0a7.dist-info/RECORD

@@ -1,13 +0,0 @@
-dgkafka/__init__.py,sha256=fnqVZROyHXipdmhqZaa9XUjvQe795JJKFakwTndAiIw,286
-dgkafka/avro_consumer.py,sha256=Hk1_vexqd5F2g2D0IC2n1HAwXR-cQrbYDsSYr616ASY,2875
-dgkafka/avro_producer.py,sha256=51p9GeavU3M21LfQZuidEkhIpzA-KGQOXhB0xMfNUus,4247
-dgkafka/config.py,sha256=JUoN0euTjPc1upnRkeDM_c3yIomJ0AI4zkcKlQhDYew,4278
-dgkafka/consumer.py,sha256=OSryUsowjpPA4ZbTSs3dwWgIgd2HOAdGeqM-Y1R1XLk,4427
-dgkafka/errors.py,sha256=PaH46tXameS--hrrUXKhQkZlBHvMSMPmjhVeRkmFvV0,95
-dgkafka/json_consumer.py,sha256=7Gzn7C9WpyCTPDV6eFDugAx5gC9vdV-HrTh3Nv--zIE,1152
-dgkafka/producer.py,sha256=4tKIYUs1zlrie5TOxnL0J38BON2HreOLf2rYW1hVrIs,5085
-dgkafka-1.0.0a7.dist-info/licenses/LICENSE,sha256=pAZXnNE2dxxwXFIduGyn1gpvPefJtUYOYZOi3yeGG94,1068
-dgkafka-1.0.0a7.dist-info/METADATA,sha256=Y2vrwwJjreIXFmfutKATCyQx0oocAAH0bpL484l5gr0,6061
-dgkafka-1.0.0a7.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dgkafka-1.0.0a7.dist-info/top_level.txt,sha256=GyNrxOh7IPdL0t2SxH8DWxg3fUma-ezQ1Kz4zIr2B7U,8
-dgkafka-1.0.0a7.dist-info/RECORD,,