dgkafka 1.0.0a0__py3-none-any.whl → 1.0.0a1__py3-none-any.whl

dgkafka/__init__.py

@@ -1,5 +1,5 @@
-from consumer import KafkaConsumer
-from avro_consumer import AvroKafkaConsumer
-from producer import KafkaProducer
-from avro_producer import AvroKafkaProducer
-from json_consumer import JsonKafkaConsumer
+from .consumer import KafkaConsumer
+from .avro_consumer import AvroKafkaConsumer
+from .producer import KafkaProducer
+from .avro_producer import AvroKafkaProducer
+from .json_consumer import JsonKafkaConsumer

dgkafka/avro_consumer.py

@@ -1,17 +1,20 @@
 from typing import Any, Iterator
 
 from dgkafka.consumer import KafkaConsumer
-from dglog import Logger
+
 from confluent_kafka import Message
 from confluent_kafka.avro import AvroConsumer
 from confluent_kafka.avro.serializer import SerializerError
 from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient
 
+import logging
+import dglog
+
 
 class AvroKafkaConsumer(KafkaConsumer):
     """Kafka consumer with Avro schema support using Schema Registry."""
 
-    def __init__(self, schema_registry_url: str, logger_: Logger | None = None, **configs: Any) -> None:
+    def __init__(self, logger_: logging.Logger | dglog.Logger | None = None, **configs: Any) -> None:
         """
         Initialize Avro consumer.
 
@@ -20,8 +23,8 @@ class AvroKafkaConsumer(KafkaConsumer):
             logger_: Optional logger instance
             configs: Kafka consumer configuration
         """
-        self.schema_registry_url = schema_registry_url
-        self.schema_registry_client = CachedSchemaRegistryClient({'url': schema_registry_url})
+        self.schema_registry_url = configs.get('schema.registry.url')
+        self.schema_registry_client = CachedSchemaRegistryClient({'url': self.schema_registry_url})
         super().__init__(logger_=logger_, **configs)
 
     def _init_consumer(self, **configs: Any) -> None:
@@ -35,7 +38,7 @@ class AvroKafkaConsumer(KafkaConsumer):
             self.logger.error(f"[x] Failed to initialize Avro consumer: {ex}")
             raise
 
-    def consume(self, num_messages: int = 1, timeout: float = 10.0, **kwargs: Any) -> Iterator[dict[str, Any] | None, Message]:
+    def consume(self, num_messages: int = 1, timeout: float = 10.0, **kwargs: Any) -> Iterator[str | bytes | Message | None]:
         """
         Consume Avro-encoded messages.
 
@@ -50,8 +53,9 @@ class AvroKafkaConsumer(KafkaConsumer):
         consumer = self._ensure_consumer()
 
         for _ in range(num_messages):
+            msg = self._consume(consumer, timeout)
             try:
-                if (msg := self._consume(consumer, timeout)) is None:
+                if msg is None:
                     continue
                 yield msg.value()
             except SerializerError as e:

dgkafka/avro_producer.py

@@ -3,7 +3,8 @@ from confluent_kafka.avro import AvroProducer
 from confluent_kafka.avro.serializer import SerializerError
 from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient
 
-from dglog import Logger
+import dglog
+import logging
 
 from dgkafka.producer import KafkaProducer
 
@@ -13,10 +14,9 @@ class AvroKafkaProducer(KafkaProducer):
 
     def __init__(
             self,
-            schema_registry_url: str,
-            default_key_schema: Optional[Dict[str, Any]] = None,
-            default_value_schema: Optional[Dict[str, Any]] = None,
-            logger_: Optional[Logger] = None,
+            default_key_schema: str | None = None,
+            default_value_schema: str | None = None,
+            logger_: logging.Logger | dglog.Logger | None = None,
             **configs: Any
     ) -> None:
         """
@@ -29,17 +29,17 @@ class AvroKafkaProducer(KafkaProducer):
             logger_: Optional logger instance
             configs: Kafka producer configuration
         """
-        self.schema_registry_url = schema_registry_url
+        self.schema_registry_url = configs.get('schema.registry.url')
+        assert self.schema_registry_url is not None, "schema.registry.url is required"
+
         self.default_key_schema = default_key_schema
         self.default_value_schema = default_value_schema
-        self.schema_registry_client = CachedSchemaRegistryClient({'url': schema_registry_url})
+        self.schema_registry_client = CachedSchemaRegistryClient({'url': self.schema_registry_url})
         super().__init__(logger_=logger_, **configs)
 
     def _init_producer(self, **configs: Any) -> None:
         """Initialize AvroProducer instance."""
         try:
-            # AvroProducer requires schema registry in config
-            configs['schema.registry.url'] = self.schema_registry_url
             self.producer = AvroProducer(
                 config=configs,
                 default_key_schema=self.default_key_schema,
@@ -53,12 +53,12 @@ class AvroKafkaProducer(KafkaProducer):
     def produce(
             self,
             topic: str,
-            value: Union[Dict[str, Any], Any],
-            key: Optional[Union[Dict[str, Any], str]] = None,
-            value_schema: Optional[Dict[str, Any]] = None,
-            key_schema: Optional[Dict[str, Any]] = None,
-            partition: Optional[int] = None,
-            headers: Optional[Dict[str, bytes]] = None,
+            value: dict[str, Any] | Any,
+            key: dict[str, Any] | str | None  = None,
+            value_schema: dict[str, Any] | None = None,
+            key_schema: dict[str, Any] | None = None,
+            partition: int | None = None,
+            headers: dict[str, bytes] | None = None,
             flush: bool = True
     ) -> None:
         """

dgkafka/config.py

@@ -0,0 +1,89 @@
+from typing import Literal, Optional
+from pydantic import BaseModel, Field
+
+
+class KafkaConfig(BaseModel):
+    bootstrap_servers: str = Field(..., alias="bootstrap.servers",
+                                 description="Comma-separated list of broker addresses")
+    security_protocol: Optional[str] = Field(default="SSL", alias="security.protocol",
+                                           description="Protocol used to communicate with brokers")
+    ssl_ca_location: Optional[str] = Field(default=None, alias="ssl.ca.location",
+                                         description="Path to CA certificate file")
+    ssl_certificate_location: Optional[str] = Field(default=None, alias="ssl.certificate.location",
+                                                  description="Path to client certificate file")
+    ssl_key_location: Optional[str] = Field(default=None, alias="ssl.key.location",
+                                          description="Path to client private key file")
+    ssl_certificate_verification: Optional[bool] = Field(default=True,
+                                                       alias="enable.ssl.certificate.verification",
+                                                       description="Enable SSL certificate verification")
+    ssl_endpoint_identification_algorithm: Optional[str] = Field(default="https",
+                                                               alias="ssl.endpoint.identification.algorithm",
+                                                               description="Endpoint identification algorithm")
+
+    class Config:
+        allow_population_by_field_name = True
+        extra = "forbid"
+
+
+class ConsumerConfig(KafkaConfig):
+    group_id: str = Field(..., alias="group.id",
+                         description="Consumer group identifier")
+    enable_auto_commit: bool = Field(default=False, alias="enable.auto.commit",
+                                   description="Automatically commit offsets periodically")
+    auto_offset_reset: Literal["earliest", "latest"] = Field(default="earliest",
+                                                           alias="auto.offset.reset",
+                                                           description="Reset policy when no offset is available")
+    max_poll_records: Optional[int] = Field(default=500, alias="max.poll.records",
+                                          description="Maximum records per poll")
+    session_timeout_ms: int = Field(default=10000, alias="session.timeout.ms",
+                                  description="Timeout for consumer session")
+
+
+class ProducerConfig(KafkaConfig):
+    acks: Literal["all", "-1", "0", "1"] = Field(default="all",
+                                                description="Number of acknowledgments")
+    retries: int = Field(default=3, description="Number of retries on failure")
+    linger_ms: int = Field(default=0, alias="linger.ms",
+                         description="Delay in milliseconds to wait for messages in the buffer")
+    compression_type: Optional[str] = Field(default=None, alias="compression.type",
+                                          description="Compression codec to use")
+    batch_size: int = Field(default=16384, alias="batch.size",
+                          description="Batch size in bytes")
+
+
+class AvroConfigMixin(BaseModel):
+    schema_registry_url: str = Field(..., alias="schema.registry.url",
+                                   description="URL of Schema Registry")
+    schema_registry_ssl_ca_location: Optional[str] = Field(default=None,
+                                                         alias="schema.registry.ssl.ca.location",
+                                                         description="Schema Registry CA certificate path")
+    auto_register_schemas: bool = Field(default=True, alias="auto.register.schemas",
+                                      description="Automatically register schemas")
+
+
+class AvroConsumerConfig(ConsumerConfig, AvroConfigMixin):
+    use_latest_version: bool = Field(default=True, alias="use.latest.version",
+                                   description="Use latest schema version")
+    skip_known_types: bool = Field(default=False, alias="skip.known.types",
+                                 description="Skip known types during deserialization")
+
+
+class AvroProducerConfig(ProducerConfig, AvroConfigMixin):
+    value_subject_name_strategy: Optional[str] = Field(default=None,
+                                                     alias="value.subject.name.strategy",
+                                                     description="Strategy for subject name generation")
+    key_subject_name_strategy: Optional[str] = Field(default=None,
+                                                   alias="key.subject.name.strategy",
+                                                   description="Strategy for key subject name generation")
+
+
+class JsonConsumerConfig(ConsumerConfig):
+    json_deserializer: Optional[str] = Field(default=None,
+                                           description="Custom JSON deserializer function")
+    encoding: str = Field(default="utf-8", description="Message encoding")
+
+
+class JsonProducerConfig(ProducerConfig):
+    json_serializer: Optional[str] = Field(default=None,
+                                         description="Custom JSON serializer function")
+    encoding: str = Field(default="utf-8", description="Message encoding")

dgkafka/consumer.py

@@ -1,17 +1,21 @@
 from typing import Literal, Iterator, Any
 from dgkafka.errors import ConsumerNotSetError
-from dglog import Logger
-from confluent_kafka import Consumer, KafkaError, KafkaException, Message, TopicPartition
+
+from confluent_kafka import Consumer, KafkaException, Message, TopicPartition
 from confluent_kafka import OFFSET_STORED, OFFSET_BEGINNING, OFFSET_END
 
+import logging
+import dglog
+
 OffsetType = Literal[OFFSET_STORED, OFFSET_BEGINNING, OFFSET_END] | int
 
 
 class KafkaConsumer:
-    def __init__(self, logger_: Logger | None = None, **configs: Any) -> None:
+    def __init__(self, logger_: logging.Logger | dglog.Logger | None = None, **configs: Any) -> None:
         self.consumer: Consumer | None = None
-        self.logger = logger_ if logger_ else Logger()
-        self.logger.auto_configure()
+        self.logger = logger_ if logger_ else dglog.Logger()
+        if isinstance(self.logger, dglog.Logger):
+            self.logger.auto_configure()
         self._init_consumer(**configs)
 
     def _init_consumer(self, **configs: Any) -> None:
@@ -84,10 +88,10 @@ class KafkaConsumer:
     def _consume(self, consumer: Consumer, timeout: float) -> Message | None:
         msg = consumer.poll(timeout)
         if msg is None:
-            return
+            return None
         if msg.error():
             self.logger.error(f"Consumer error: {msg.error()}")
-            return
+            return None
         self.logger.info(f"[<] Received message from {msg.topic()} [partition {msg.partition()}, offset {msg.offset()}]")
         self.logger.debug(f"[*] Message content: {msg.value()}")
         return msg

dgkafka/producer.py

@@ -1,12 +1,15 @@
 import uuid
 from typing import Optional, Any
-from dglog import Logger
+
 from confluent_kafka import Producer, Message
 from dgkafka.errors import ProducerNotSetError
 
+import logging
+import dglog
+
 
 class KafkaProducer:
-    def __init__(self, logger_: Logger | None = None, **configs: Any) -> None:
+    def __init__(self, logger_: logging.Logger | dglog.Logger | None = None, **configs: Any) -> None:
         """Initialize Kafka producer.
 
         Args:
@@ -14,8 +17,9 @@ class KafkaProducer:
             configs: Kafka producer configuration
         """
         self.producer: Producer | None = None
-        self.logger = logger_ if logger_ else Logger()
-        self.logger.auto_configure()
+        self.logger = logger_ if logger_ else dglog.Logger()
+        if isinstance(self.logger, dglog.Logger):
+            self.logger.auto_configure()
         self._init_producer(**configs)
 
     def _init_producer(self, **configs: Any) -> None:

{dgkafka-1.0.0a0.dist-info → dgkafka-1.0.0a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dgkafka
-Version: 1.0.0a0
+Version: 1.0.0a1
 Summary: Kafka clients
 Home-page: https://gitlab.com/gng-group/dgkafka.git
 Author: Malanris
@@ -8,17 +8,29 @@ Author-email: Roman Rasputin <admin@roro.su>
 License: MIT License
 Project-URL: Homepage, https://gitlab.com/gng-group/dgkafka
 Project-URL: BugTracker, https://gitlab.com/gng-group/dgkafka/issues
-Keywords: logging,logger,rotation,timed,advanced
+Keywords: kafka,client,confluent,avro,fastapi,logging
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Logging
-Requires-Python: >=3.7
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: confluent-kafka>=2.1.1
+Requires-Dist: dglog>=1.0.0
+Provides-Extra: avro
+Requires-Dist: requests; extra == "avro"
+Requires-Dist: fastavro<2; extra == "avro"
+Requires-Dist: avro<2,>=1.11.1; extra == "avro"
+Requires-Dist: attrs; extra == "avro"
+Requires-Dist: cachetools; extra == "avro"
+Requires-Dist: httpx>=0.26; extra == "avro"
+Requires-Dist: authlib; extra == "avro"
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

dgkafka-1.0.0a1.dist-info/RECORD

@@ -0,0 +1,13 @@
+dgkafka/__init__.py,sha256=bHvunrggolv8quKYZAK8sntcFEBHcQ47PQWH8kjM2Qw,206
+dgkafka/avro_consumer.py,sha256=TG8l07SDspk9sZkdnyCRYBWQ5qS5Swdrkcf48izPtFs,2924
+dgkafka/avro_producer.py,sha256=5OPhg-Ulm8gmimjC2MlUqlnbsGjcn2Mf159PEWqt-Ac,4252
+dgkafka/config.py,sha256=-rsJO_xAEEFJ9wZtN0LnbxHKpvruC8uOkLipI77a4zk,5425
+dgkafka/consumer.py,sha256=OSryUsowjpPA4ZbTSs3dwWgIgd2HOAdGeqM-Y1R1XLk,4427
+dgkafka/errors.py,sha256=PaH46tXameS--hrrUXKhQkZlBHvMSMPmjhVeRkmFvV0,95
+dgkafka/json_consumer.py,sha256=7Gzn7C9WpyCTPDV6eFDugAx5gC9vdV-HrTh3Nv--zIE,1152
+dgkafka/producer.py,sha256=4tKIYUs1zlrie5TOxnL0J38BON2HreOLf2rYW1hVrIs,5085
+dgkafka-1.0.0a1.dist-info/licenses/LICENSE,sha256=pAZXnNE2dxxwXFIduGyn1gpvPefJtUYOYZOi3yeGG94,1068
+dgkafka-1.0.0a1.dist-info/METADATA,sha256=ypN62v2UoQMRDL9mUZNWUElnQEU2EfafBezBuQnc9pc,5837
+dgkafka-1.0.0a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dgkafka-1.0.0a1.dist-info/top_level.txt,sha256=GyNrxOh7IPdL0t2SxH8DWxg3fUma-ezQ1Kz4zIr2B7U,8
+dgkafka-1.0.0a1.dist-info/RECORD,,

dgkafka-1.0.0a0.dist-info/RECORD

@@ -1,12 +0,0 @@
-dgkafka/__init__.py,sha256=2hPW2KtgErppTV4_5NeKUVmLiic4evF1eRoqmXCLpwU,201
-dgkafka/avro_consumer.py,sha256=5nWQwY6ynX_h8KGSdZ2LCL9BlXcyHlRqjwnFP66zccE,2891
-dgkafka/avro_producer.py,sha256=Vn9-t30U_pj81clSnadXCPYoXZJ_O_EuRZKa2yYlmrU,4345
-dgkafka/consumer.py,sha256=pVqXpAR7pA8JyYmg49PdOct0_fou5RmQmrGTtP8cd7o,4341
-dgkafka/errors.py,sha256=PaH46tXameS--hrrUXKhQkZlBHvMSMPmjhVeRkmFvV0,95
-dgkafka/json_consumer.py,sha256=7Gzn7C9WpyCTPDV6eFDugAx5gC9vdV-HrTh3Nv--zIE,1152
-dgkafka/producer.py,sha256=-901NGP0szc6KQ1VOKtoq0zScIME9Bg6VzgRgBBbYFM,4997
-dgkafka-1.0.0a0.dist-info/licenses/LICENSE,sha256=pAZXnNE2dxxwXFIduGyn1gpvPefJtUYOYZOi3yeGG94,1068
-dgkafka-1.0.0a0.dist-info/METADATA,sha256=vbR-EAldSdreDK5W9rDLISmiSHBLjWOgLx9cfjT6a1o,5340
-dgkafka-1.0.0a0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dgkafka-1.0.0a0.dist-info/top_level.txt,sha256=GyNrxOh7IPdL0t2SxH8DWxg3fUma-ezQ1Kz4zIr2B7U,8
-dgkafka-1.0.0a0.dist-info/RECORD,,