dgkafka 1.0.0__py3-none-any.whl

dgkafka/__init__.py

@@ -0,0 +1,11 @@
+from .consumer import KafkaConsumer
+from .producer import KafkaProducer
+try:
+    from .avro_consumer import AvroKafkaConsumer
+    from .avro_producer import AvroKafkaProducer
+except ImportError:
+    pass
+try:
+    from .json_consumer import JsonKafkaConsumer
+except ImportError:
+    pass

dgkafka/avro_consumer.py

@@ -0,0 +1,74 @@
+from typing import Any, Iterator
+
+from dgkafka.consumer import KafkaConsumer
+
+from confluent_kafka import Message
+from confluent_kafka.avro import AvroConsumer
+from confluent_kafka.avro.serializer import SerializerError
+from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient
+
+import logging
+import dglog
+
+
+class AvroKafkaConsumer(KafkaConsumer):
+    """Kafka consumer with Avro schema support using Schema Registry."""
+
+    def __init__(self, logger_: logging.Logger | dglog.Logger | None = None, **configs: Any) -> None:
+        """
+        Initialize Avro consumer.
+
+        Args:
+            schema_registry_url: URL of Schema Registry
+            logger_: Optional logger instance
+            configs: Kafka consumer configuration
+        """
+        self.schema_registry_url = configs.get('schema.registry.url')
+        assert self.schema_registry_url is not None, "schema.registry.url is required"
+
+        self.schema_registry_client = CachedSchemaRegistryClient(url=self.schema_registry_url)
+        super().__init__(logger_=logger_, **configs)
+
+    def _init_consumer(self, **configs: Any) -> None:
+        """Initialize AvroConsumer instance."""
+        try:
+            self.consumer = AvroConsumer(configs)
+            self.logger.info("[*] Avro consumer initialized successfully")
+        except Exception as ex:
+            self.logger.error(f"[x] Failed to initialize avro consumer: {ex}")
+            raise
+
+    def consume(self, num_messages: int = 1, timeout: float = 1.0, decode_: bool = False, **kwargs: Any) -> Iterator[str | bytes | Message | None]:
+        """
+        Consume Avro-encoded messages.
+
+        Args:
+            num_messages: Maximum number of messages to consume
+            timeout: Poll timeout in seconds
+            kwargs: Additional arguments
+
+        Yields:
+            Deserialized Avro messages as dictionaries or Message objects on error
+        """
+        consumer = self._ensure_consumer()
+
+        for _ in range(num_messages):
+            msg = self._consume(consumer, timeout)
+            try:
+                if msg is None:
+                    continue
+                yield msg.value() if decode_ else msg
+            except SerializerError as e:
+                self.logger.error(f"[x] Avro deserialization failed: {e}")
+                yield msg  # Return raw message on deserialization error
+            except Exception as ex:
+                self.logger.error(f"[!] Unexpected error: {ex}")
+                continue
+
+    def get_schema(self, subject: str, version: int = 1) -> dict[str, Any]:
+        """Get Avro schema from Schema Registry."""
+        return self.schema_registry_client.get_schema(subject, version)
+
+    def get_latest_schema(self, subject: str) -> dict[str, Any]:
+        """Get latest version of schema for given subject."""
+        return self.schema_registry_client.get_latest_schema(subject)[1]

dgkafka/avro_producer.py

@@ -0,0 +1,138 @@
+from typing import Optional, Union, Dict, Any
+from confluent_kafka.avro import AvroProducer
+from confluent_kafka.avro.serializer import SerializerError
+from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient
+
+import dglog
+import logging
+
+from dgkafka.producer import KafkaProducer
+
+
+class AvroKafkaProducer(KafkaProducer):
+    """Kafka producer with Avro schema support using Schema Registry."""
+
+    def __init__(
+            self,
+            default_key_schema: str | None = None,
+            default_value_schema: str | None = None,
+            logger_: logging.Logger | dglog.Logger | None = None,
+            **configs: Any
+    ) -> None:
+        """
+        Initialize Avro producer.
+
+        Args:
+            schema_registry_url: URL of Schema Registry
+            default_key_schema: Default Avro schema for message keys
+            default_value_schema: Default Avro schema for message values
+            logger_: Optional logger instance
+            configs: Kafka producer configuration
+        """
+        self.schema_registry_url = configs.get('schema.registry.url')
+        assert self.schema_registry_url is not None, "schema.registry.url is required"
+
+        self.default_key_schema = default_key_schema
+        self.default_value_schema = default_value_schema
+        self.schema_registry_client = CachedSchemaRegistryClient(url=self.schema_registry_url)
+        super().__init__(logger_=logger_, **configs)
+
+    def _init_producer(self, **configs: Any) -> None:
+        """Initialize AvroProducer instance."""
+        try:
+            self.producer = AvroProducer(
+                config=configs,
+                default_key_schema=self.default_key_schema,
+                default_value_schema=self.default_value_schema
+            )
+            self.logger.info("[*] Avro producer initialized successfully")
+        except Exception as ex:
+            self.logger.error(f"[x] Failed to initialize avro producer: {ex}")
+            raise
+
+    def produce(
+            self,
+            topic: str,
+            value: dict[str, Any] | Any,
+            key: dict[str, Any] | str | None  = None,
+            value_schema: dict[str, Any] | None = None,
+            key_schema: dict[str, Any] | None = None,
+            partition: int | None = None,
+            headers: dict[str, bytes] | None = None,
+            flush: bool = True
+    ) -> None:
+        """
+        Produce Avro-encoded message to Kafka.
+
+        Args:
+            topic: Target topic name
+            value: Message value (must match Avro schema)
+            key: Message key (optional)
+            value_schema: Avro schema for message value (optional)
+            key_schema: Avro schema for message key (optional)
+            partition: Specific partition (optional)
+            headers: Message headers (optional)
+            flush: Immediately flush after producing (default: True)
+        """
+        producer = self._ensure_producer()
+        producer.poll(0)
+
+        self._delivery_status['success'] = None
+
+        # Prepare headers
+        headers_list = None
+        if headers:
+            headers_list = [(k, v if isinstance(v, bytes) else str(v).encode('utf-8'))
+                            for k, v in headers.items()]
+
+        try:
+            if not partition:
+                producer.produce(
+                    topic=topic,
+                    value=value,
+                    value_schema=value_schema,
+                    key=key,
+                    key_schema=key_schema,
+                    on_delivery=self.delivery_report,
+                    headers=headers_list
+                )
+            else:
+                producer.produce(
+                    topic=topic,
+                    value=value,
+                    value_schema=value_schema,
+                    key=key,
+                    key_schema=key_schema,
+                    partition=partition,
+                    on_delivery=self.delivery_report,
+                    headers=headers_list
+                )
+
+            if flush:
+                remaining = producer.flush(timeout)
+                if remaining > 0:
+                    self.logger.warning(f"[!] {remaining} messages remain undelivered after flush timeout")
+                    return False
+
+            # Если flush=True, статус должен быть установлен к этому моменту
+            if flush and self._delivery_status['success'] is not None:
+                return self._delivery_status['success']
+
+            # Если flush=False, мы не можем гарантировать доставку, возвращаем True
+            # (так как технически ошибки пока нет)
+            return True
+
+        except SerializerError as ex:
+            self.logger.error(f"[x] Avro serialization failed: {ex}")
+            return False
+        except Exception as ex:
+            self.logger.error(f"[x] Failed to produce Avro message: {ex}")
+            return False
+
+    def get_schema(self, subject: str, version: int = 1) -> Dict[str, Any]:
+        """Get Avro schema from Schema Registry."""
+        return self.schema_registry_client.get_schema(subject, version)
+
+    def get_latest_schema(self, subject: str) -> Dict[str, Any]:
+        """Get latest version of schema for given subject."""
+        return self.schema_registry_client.get_latest_schema(subject)[1]

dgkafka/config.py

@@ -0,0 +1,84 @@
+from typing import Dict, Any, Optional, Literal
+from pydantic import BaseModel, Field, ConfigDict
+from enum import Enum
+
+class SecurityProtocol(str, Enum):
+    PLAINTEXT = "PLAINTEXT"
+    SSL = "SSL"
+    SASL_PLAINTEXT = "SASL_PLAINTEXT"
+    SASL_SSL = "SASL_SSL"
+
+class KafkaConfig(BaseModel):
+    """Base configuration for all Kafka clients"""
+    bootstrap_servers: str = Field(..., alias="bootstrap.servers")
+    security_protocol: Literal["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"] = Field(default="SSL",
+                                                alias="security.protocol")
+    ssl_ca_location: Optional[str] = Field(default=None, alias="ssl.ca.location")
+    ssl_certificate_location: Optional[str] = Field(default=None, alias="ssl.certificate.location")
+    ssl_key_location: Optional[str] = Field(default=None, alias="ssl.key.location")
+    ssl_endpoint_identification_algorithm: Optional[str] = Field(default=None,
+                                                                 alias="ssl.endpoint.identification.algorithm")
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        extra="forbid",
+        protected_namespaces=()
+    )
+
+    def get(self) -> Dict[str, Any]:
+        """Get config in format suitable for confluent_kafka"""
+        return self.model_dump(by_alias=True, exclude_none=True)
+
+    @classmethod
+    def set(cls, config_dict: Dict[str, Any]) -> "KafkaConfig":
+        """Create config from dictionary"""
+        return cls(**config_dict)
+
+class ConsumerConfig(KafkaConfig):
+    """Base consumer configuration"""
+    group_id: str = Field(..., alias="group.id")
+    enable_auto_commit: bool = Field(default=False, alias="enable.auto.commit")
+    auto_offset_reset: Literal["earliest", "latest"] = Field(
+        default="earliest", alias="auto.offset.reset")
+    session_timeout_ms: int = Field(default=10000, alias="session.timeout.ms")
+    max_poll_interval_ms: int = Field(default=300000, alias="max.poll.interval.ms")
+
+class ProducerConfig(KafkaConfig):
+    """Base producer configuration"""
+    acks: Literal["all", "0", "1"] = Field(default="all")
+    retries: int = Field(default=0)
+    compression_type: str = Field(default="none", alias="compression.type")
+    batch_size: int = Field(default=16384, alias="batch.size")
+    linger_ms: int = Field(default=0, alias="linger.ms")
+
+
+class AvroConfigMixin:
+    schema_registry_url: str = Field(..., alias="schema.registry.url")
+    schema_registry_ssl_ca_location: Optional[str] = Field(
+        default=None, alias="schema.registry.ssl.ca.location")
+    schema_registry_ssl_certificate_location: Optional[str] = Field(
+        default=None, alias="schema.registry.ssl.certificate.location")
+    schema_registry_ssl_key_location: Optional[str] = Field(
+        default=None, alias="schema.registry.ssl.key.location")
+
+
+class AvroConsumerConfig(ConsumerConfig, AvroConfigMixin):
+    """Avro consumer configuration with Schema Registry support"""
+
+    @classmethod
+    def set(cls, config_dict: Dict[str, Any]) -> "AvroConsumerConfig":
+        """Create from dictionary with Schema Registry validation"""
+        if "schema.registry.url" not in config_dict:
+            raise ValueError("schema.registry.url is required for AvroConsumer")
+        return cls(**config_dict)
+
+class AvroProducerConfig(ProducerConfig, AvroConfigMixin):
+    """Avro producer configuration with Schema Registry support"""
+    max_schemas_per_subject: int = Field(default=1000, alias="max.schemas.per.subject")
+
+    @classmethod
+    def set(cls, config_dict: Dict[str, Any]) -> "AvroProducerConfig":
+        """Create from dictionary with Schema Registry validation"""
+        if "schema.registry.url" not in config_dict:
+            raise ValueError("schema.registry.url is required for AvroProducer")
+        return cls(**config_dict)

dgkafka/consumer.py

@@ -0,0 +1,197 @@
+from typing import Literal, Iterator, Any
+from dgkafka.errors import ConsumerNotSetError
+
+from confluent_kafka import Consumer, KafkaException, Message, TopicPartition
+from confluent_kafka import OFFSET_STORED, OFFSET_BEGINNING, OFFSET_END
+
+import logging
+import dglog
+
+OffsetType = Literal[OFFSET_STORED, OFFSET_BEGINNING, OFFSET_END] | int
+
+
+class KafkaConsumer:
+    def __init__(self, logger_: logging.Logger | dglog.Logger | None = None, **configs: Any) -> None:
+        self.consumer: Consumer | None = None
+        self.logger = logger_ if logger_ else dglog.Logger()
+        if isinstance(self.logger, dglog.Logger):
+            self.logger.auto_configure()
+        self._init_consumer(**configs)
+
+    def _init_consumer(self, **configs: Any) -> None:
+        """Internal method to initialize consumer"""
+        try:
+            self.consumer = Consumer(configs)
+            self.logger.info("[*] Consumer initialized successfully")
+        except KafkaException as ex:
+            self.logger.error(f"[x] Failed to initialize consumer: {ex}")
+            raise
+
+    def close(self) -> None:
+        """Safely close the consumer"""
+        if self.consumer is not None:
+            try:
+                self.consumer.close()
+                self.logger.info("[*] Consumer closed successfully")
+            except KafkaException as ex:
+                self.logger.error(f"[x] Error closing consumer: {ex}")
+                raise
+            finally:
+                self.consumer = None
+
+    def __enter__(self):
+        """Context manager entry point"""
+        if self.consumer is None:
+            self._init_consumer()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit point"""
+        self.close()
+
+    def _ensure_consumer(self) -> Consumer:
+        """Ensure consumer is initialized"""
+        if self.consumer is None:
+            raise ConsumerNotSetError('[!] Consumer not initialized!')
+        return self.consumer
+
+    def subscribe(self, topics: str | list[str], partition: int | None = None,
+                  offset: OffsetType = OFFSET_STORED) -> None:
+        """Subscribe to topics"""
+        consumer = self._ensure_consumer()
+
+        if partition is not None and offset != OFFSET_STORED:
+            topic_list = [topics] if isinstance(topics, str) else topics
+            for topic in topic_list:
+                self._assign_topic_partition(topic, partition, offset)
+        else:
+            topics_list = [topics] if isinstance(topics, str) else topics
+            consumer.subscribe(topics_list, on_assign=self.on_assign, on_revoke=self.on_revoke)
+
+    def on_assign(self, consumer, partitions):
+        self.kafka_status = "UP"
+        for topic in {p.topic for p in partitions}:
+            new = {p.partition for p in partitions if p.topic == topic}
+            self.logger.debug(f"[@] on_assign {topic} {new if new else '{}'}")
+            old = {p.partition for p in consumer.assignment() if p.topic == topic}
+            old.update(new)
+            self.logger.info(f"[*] Assigned  {topic} {old if old else '{}'}")
+
+    def on_revoke(self, consumer, partitions):
+        for topic in {p.topic for p in partitions}:
+            new = {p.partition for p in partitions if p.topic == topic}
+            self.logger.debug(f"[@] on_revoke {topic} {new if new else '{}'}")
+            old = {p.partition for p in consumer.assignment() if p.topic == topic}
+            old.difference_update(new)
+            self.logger.info(f"[*] Assigned  {topic} {old if old else '{}'}")
+
+    def _assign_topic_partition(self, topic: str, partition: int, offset: OffsetType) -> None:
+        """Assign to specific partition"""
+        consumer = self._ensure_consumer()
+        topic_partition = TopicPartition(topic, partition, offset)
+        consumer.assign([topic_partition])
+        consumer.seek(topic_partition)
+        self.logger.info(f"[*] Assigned to topic '{topic}' partition {partition} with offset {offset}")
+
+    def consume(self, num_messages: int = 1, timeout: float = 1.0, decode_: bool = False) -> Iterator[Message | str]:
+        """Consume messages"""
+        consumer = self._ensure_consumer()
+
+        for _ in range(num_messages):
+            if (msg := self._consume(consumer, timeout)) is None:
+                continue
+            yield msg.value().decode('utf-8') if decode_ else msg
+
+    def _consume(self, consumer: Consumer, timeout: float) -> Message | None:
+        msg = consumer.poll(timeout)
+        if msg is None:
+            return None
+        if msg.error():
+            self.logger.error(f"[x] Consumer error: {msg.error()}")
+            return None
+        self.logger.info(f"[<] Received message from {msg.topic()} [partition {msg.partition()}, offset {msg.offset()}]")
+        self.logger.debug(f"[*] Message content: {msg.value()}")
+        return msg
+
+    def commit(self, message: Message | None = None, offsets: list[TopicPartition] | None = None,
+               asynchronous: bool = True) -> list[TopicPartition] | None:
+        """Commit offsets to Kafka."""
+        consumer = self._ensure_consumer()
+        if message:
+            return consumer.commit(message=message, asynchronous=asynchronous)
+        elif offsets:
+            return consumer.commit(offsets=offsets, asynchronous=asynchronous)
+        return consumer.commit(asynchronous=asynchronous)
+
+    def get_subscription_info(self) -> dict:
+        """Get current subscription and assignment information.
+
+        Returns:
+            dict: Dictionary with subscription and assignment details
+            {
+                'subscribed_topics': list[str] | None,
+                'assignments': list[dict] | None,
+                'current_offsets': list[dict] | None
+            }
+        """
+        consumer = self._ensure_consumer()
+
+        try:
+            # Получаем текущие назначения (assignments)
+            assignments = consumer.assignment()
+
+            # Получаем текущие позиции (offsets)
+            current_offsets = []
+            if assignments:
+                current_offsets = [consumer.position(tp) for tp in assignments]
+
+            # Для получения подписок используем список топиков из assignments
+            subscribed_topics = list({tp.topic for tp in assignments}) if assignments else None
+
+            # Формируем информацию о назначениях
+            assignments_info = []
+            for tp in assignments:
+                assignments_info.append({
+                    'topic': tp.topic,
+                    'partition': tp.partition,
+                    'offset': tp.offset
+                })
+
+            # Формируем информацию о текущих позициях
+            offsets_info = []
+            for tp in current_offsets:
+                offsets_info.append({
+                    'topic': tp.topic,
+                    'partition': tp.partition,
+                    'offset': tp.offset
+                })
+
+            return {
+                'subscribed_topics': subscribed_topics,
+                'assignments': assignments_info if assignments_info else None,
+                'current_offsets': offsets_info if offsets_info else None
+            }
+
+        except KafkaException as ex:
+            self.logger.error(f"[x] Failed to get subscription info: {ex}")
+            raise
+
+    def log_subscription_info(self) -> None:
+        """Log current subscription and assignment information."""
+        info = self.get_subscription_info()
+
+        if info['subscribed_topics']:
+            self.logger.info(f"[*] Subscribed topics: {', '.join(info['subscribed_topics'])}")
+        else:
+            self.logger.info("[!] Not subscribed to any topics")
+
+        if info['assignments']:
+            self.logger.info("[*] Current partition assignments:")
+            for assignment in info['assignments']:
+                self.logger.info(f"    - {assignment['topic']} [partition {assignment['partition']}]")
+
+        if info['current_offsets']:
+            self.logger.info("[*] Current read positions:")
+            for offset in info['current_offsets']:
+                self.logger.info(
+                    f"    - {offset['topic']} [partition {offset['partition']}]: position {offset['offset']}")

dgkafka/errors.py

@@ -0,0 +1,6 @@
+class ProducerNotSetError(Exception):
+    pass
+
+
+class ConsumerNotSetError(Exception):
+    pass

dgkafka/json_consumer.py

@@ -0,0 +1,25 @@
+from confluent_kafka import DeserializingConsumer
+from confluent_kafka.schema_registry import SchemaRegistryClient
+from confluent_kafka.schema_registry.json_schema import JSONDeserializer
+from confluent_kafka.serialization import StringDeserializer
+from dglog import Logger
+from dgkafka.consumer import KafkaConsumer
+
+
+class JsonKafkaConsumer(KafkaConsumer):
+    def __init__(self, logger_: Logger | None = None, **configs):
+        self.consumer: DeserializingConsumer | None = None
+        self.schema_registry = {'url': configs.pop('schema.registry.url')}
+        self.schema_client = SchemaRegistryClient(self.schema_registry)
+        self.deserializer = JSONDeserializer(schema_str=None, schema_registry_client=self.schema_client)
+
+        super(JsonKafkaConsumer, self).__init__(logger_, **configs)
+
+    def init_consumer(self, logger_: Logger | None = None, **configs):
+        consumer_conf = {
+            **configs,
+            'key.deserializer': StringDeserializer('utf_8'),
+            'value.deserializer': self.deserializer
+        }
+        self.logger = logger_ or Logger()
+        self.consumer = DeserializingConsumer(consumer_conf)

dgkafka/producer.py

@@ -0,0 +1,180 @@
+import uuid
+from typing import Optional, Any
+
+from datetime import datetime, date
+
+from confluent_kafka import Producer, Message
+from dgkafka.errors import ProducerNotSetError
+
+import logging
+import dglog
+
+
+class KafkaProducer:
+    def __init__(self, logger_: logging.Logger | dglog.Logger | None = None, **configs: Any) -> None:
+        """Initialize Kafka producer.
+
+        Args:
+            logger_: Optional logger instance
+            configs: Kafka producer configuration
+        """
+        self.producer: Producer | None = None
+        self.logger = logger_ if logger_ else dglog.Logger()
+
+        self._delivery_status = {'success': None}
+
+        if isinstance(self.logger, dglog.Logger):
+            self.logger.auto_configure()
+        self._init_producer(**configs)
+
+    def _init_producer(self, **configs: Any) -> None:
+        """Internal method to initialize producer."""
+        try:
+            self.producer = Producer(configs)
+            self.logger.info("[*] Producer initialized successfully")
+        except Exception as ex:
+            self.logger.error(f"[x] Failed to initialize producer: {ex}")
+            raise
+
+    def close(self) -> None:
+        """Close the producer connection."""
+        if self.producer is not None:
+            try:
+                self.producer.flush()
+                self.logger.info("[*] Producer closed successfully")
+            except Exception as ex:
+                self.logger.error(f"[x] Error closing producer: {ex}")
+                raise
+            finally:
+                self.producer = None
+
+    def __enter__(self):
+        """Context manager entry point."""
+        if self.producer is None:
+            self._init_producer()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit point."""
+        self.close()
+
+    def _ensure_producer(self) -> Producer:
+        """Ensure producer is initialized."""
+        if self.producer is None:
+            raise ProducerNotSetError('[!] Producer not initialized!')
+        return self.producer
+
+    def delivery_report(self, err: Optional[Any], msg: Message) -> None:
+        """Delivery callback for produced messages.
+
+        Args:
+            err: Error object if delivery failed
+            msg: Delivered message object
+        """
+        if err is not None:
+            self.logger.error(f"[x] Message delivery failed: {err}")
+            self.logger.debug(f"[~] Failed message details: {msg}")
+            self._delivery_status['success'] = False
+        else:
+            self.logger.info(
+                f"[>] Message delivered to {msg.topic()} [partition {msg.partition()}, offset {msg.offset()}]")
+            self._delivery_status['success'] = True
+
+    def produce(
+            self,
+            topic: str,
+            message: str | bytes | dict[str, Any],
+            key: str | None = None,
+            partition: int | None = None,
+            headers: dict[str, bytes] | None = None,
+            flush: bool = True
+    ) -> bool:
+        """Produce a message to Kafka.
+
+        Args:
+            topic: Target topic name
+            message: Message content (str, bytes or dict)
+            key: Message key (optional)
+            partition: Specific partition (optional)
+            headers: Message headers (optional)
+            flush: Immediately flush after producing (default: True)
+        """
+        producer = self._ensure_producer()
+        producer.poll(0)
+
+        self._delivery_status['success'] = None
+
+        # Generate key if not provided
+        key = key if key is not None else str(uuid.uuid4())
+        key_bytes = key.encode('utf-8')
+
+        # Prepare message value
+        if isinstance(message, str):
+            value = message.encode('utf-8')
+        elif isinstance(message, bytes):
+            value = message
+        else:  # Assume dict-like object
+            try:
+                import json
+                dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime) or isinstance(obj, date) else None
+                value = json.dumps(message, ensure_ascii=False, default=dthandler, indent=4).encode('utf-8')
+            except Exception as ex:
+                self.logger.error(f"[x] Failed to serialize message: {ex}")
+                return False
+
+        # Prepare message headers
+        headers_list = None
+        if headers:
+            headers_list = [(k, v if isinstance(v, bytes) else str(v).encode('utf-8'))
+                            for k, v in headers.items()]
+
+        # Produce message
+        try:
+            if not partition:
+                producer.produce(
+                    topic=topic,
+                    value=value,
+                    key=key_bytes,
+                    on_delivery=self.delivery_report,
+                    headers=headers_list
+                )
+            else:
+                producer.produce(
+                    topic=topic,
+                    value=value,
+                    key=key_bytes,
+                    partition=partition,
+                    on_delivery=self.delivery_report,
+                    headers=headers_list
+                )
+            if flush:
+                remaining = producer.flush(1.0)  # timeout 1 second
+                if remaining > 0:
+                    self.logger.warning(f"[!] {remaining} messages remain undelivered")
+                    return False
+
+            # Если flush=True, статус должен быть установлен к этому моменту
+            if flush and self._delivery_status['success'] is not None:
+                return self._delivery_status['success']
+
+            # Если flush=False, мы не можем гарантировать доставку, возвращаем True
+            # (так как технически ошибки пока нет)
+            return True
+        except Exception as ex:
+            self.logger.error(f"[x] Failed to produce message: {ex}")
+            return False
+
+    def flush(self, timeout: float = 10.0) -> None:
+        """Wait for all messages to be delivered.
+
+        Args:
+            timeout: Maximum time to wait (seconds)
+        """
+        producer = self._ensure_producer()
+        try:
+            remaining = producer.flush(timeout)
+            if remaining > 0:
+                self.logger.warning(f"[!] {remaining} messages remain undelivered after flush timeout")
+        except Exception as ex:
+            self.logger.error(f"[x] Flush failed: {ex}")
+            raise

dgkafka-1.0.0.dist-info/METADATA

@@ -0,0 +1,239 @@
+Metadata-Version: 2.4
+Name: dgkafka
+Version: 1.0.0
+Summary: Kafka clients
+Home-page: https://gitlab.com/gng-group/dgkafka.git
+Author: Malanris
+Author-email: Roman Rasputin <admin@roro.su>
+License: MIT License
+Project-URL: Homepage, https://gitlab.com/gng-group/dgkafka
+Project-URL: BugTracker, https://gitlab.com/gng-group/dgkafka/issues
+Keywords: kafka,client,confluent,avro,fastapi,logging
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Logging
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: confluent-kafka>=2.1.1
+Requires-Dist: dglog>=1.0.0
+Requires-Dist: pydantic
+Provides-Extra: avro
+Requires-Dist: requests; extra == "avro"
+Requires-Dist: fastavro<2; extra == "avro"
+Requires-Dist: avro<2,>=1.11.1; extra == "avro"
+Requires-Dist: attrs; extra == "avro"
+Requires-Dist: cachetools; extra == "avro"
+Requires-Dist: httpx>=0.26; extra == "avro"
+Requires-Dist: authlib; extra == "avro"
+Provides-Extra: json
+Requires-Dist: pyrsistent; extra == "json"
+Requires-Dist: jsonschema; extra == "json"
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+
+# dgkafka
+
+Python package for working with Apache Kafka supporting multiple data formats.
+
+## Installation
+
+```bash
+pip install dgkafka
+```
+
+For Avro support (requires additional dependencies):
+
+```bash
+pip install dgkafka[avro]
+```
+
+For Json support (requires additional dependencies):
+
+```bash
+pip install dgkafka[json]
+```
+
+## Features
+
+- Producers and consumers for different data formats:
+  - Raw messages (bytes/strings)
+  - JSON
+  - Avro (with Schema Registry integration)
+- Robust error handling
+- Comprehensive operation logging
+- Context manager support
+- Flexible configuration
+
+## Quick Start
+
+### Basic Producer/Consumer
+
+```python
+from dgkafka import KafkaProducer, KafkaConsumer
+
+# Producer
+with KafkaProducer(bootstrap_servers='localhost:9092') as producer:
+    producer.produce('test_topic', 'Hello, Kafka!')
+
+# Consumer
+with KafkaConsumer(bootstrap_servers='localhost:9092', group_id='test_group') as consumer:
+    consumer.subscribe(['test_topic'])
+    for msg in consumer.consume():
+        print(msg.value())
+```
+
+### JSON Support
+
+```python
+from dgkafka import JsonKafkaProducer, JsonKafkaConsumer
+
+# Producer
+with JsonKafkaProducer(bootstrap_servers='localhost:9092') as producer:
+    producer.produce('json_topic', {'key': 'value'})
+
+# Consumer
+with JsonKafkaConsumer(bootstrap_servers='localhost:9092', group_id='json_group') as consumer:
+    consumer.subscribe(['json_topic'])
+    for msg in consumer.consume():
+        print(msg.value())  # Automatically deserialized JSON
+```
+
+### Avro Support
+
+```python
+from dgkafka import AvroKafkaProducer, AvroKafkaConsumer
+
+# Producer
+value_schema = {
+    "type": "record",
+    "name": "User",
+    "fields": [
+        {"name": "name", "type": "string"},
+        {"name": "age", "type": "int"}
+    ]
+}
+
+with AvroKafkaProducer(
+    schema_registry_url='http://localhost:8081',
+    bootstrap_servers='localhost:9092',
+    default_value_schema=value_schema
+) as producer:
+    producer.produce('avro_topic', {'name': 'Alice', 'age': 30})
+
+# Consumer
+with AvroKafkaConsumer(
+    schema_registry_url='http://localhost:8081',
+    bootstrap_servers='localhost:9092',
+    group_id='avro_group'
+) as consumer:
+    consumer.subscribe(['avro_topic'])
+    for msg in consumer.consume():
+        print(msg.value())  # Automatically deserialized Avro object
+```
+
+## Classes
+
+### Base Classes
+
+- `KafkaProducer` - base message producer
+- `KafkaConsumer` - base message consumer
+
+### Specialized Classes
+
+- `JsonKafkaProducer` - JSON message producer (inherits from `KafkaProducer`)
+- `JsonKafkaConsumer` - JSON message consumer (inherits from `KafkaConsumer`)
+- `AvroKafkaProducer` - Avro message producer (inherits from `KafkaProducer`)
+- `AvroKafkaConsumer` - Avro message consumer (inherits from `KafkaConsumer`)
+
+## Configuration
+
+All classes accept standard Kafka configuration parameters:
+
+```python
+config = {
+    'bootstrap.servers': 'localhost:9092',
+    'group.id': 'my_group',
+    'auto.offset.reset': 'earliest'
+}
+```
+
+Avro classes require additional parameter:
+- `schema_registry_url` - Schema Registry URL
+
+## Logging
+
+All classes use `dglog.Logger` for logging. You can provide a custom logger:
+
+```python
+from dglog import Logger
+
+logger = Logger()
+producer = KafkaProducer(logger_=logger, ...)
+```
+
+## Best Practices
+
+1. Always use context managers (`with`) for proper resource cleanup
+2. Implement error handling and retry logic for production use
+3. Pre-register Avro schemas in Schema Registry
+4. Configure appropriate `acks` and `retries` parameters for producers
+5. Monitor consumer lag and producer throughput
+
+## Advanced Usage
+
+### Custom Serialization
+
+```python
+# Custom Avro serializer
+class CustomAvroProducer(AvroKafkaProducer):
+    def _serialize_value(self, value):
+        # Custom serialization logic
+        return super()._serialize_value(value)
+```
+
+### Message Headers
+
+```python
+# Adding headers to messages
+headers = {
+    'correlation_id': '12345',
+    'message_type': 'user_update'
+}
+
+producer.produce(
+    topic='events',
+    value=message_data,
+    headers=headers
+)
+```
+
+### Error Handling
+
+```python
+from confluent_kafka import KafkaException
+
+try:
+    with AvroKafkaProducer(...) as producer:
+        producer.produce(...)
+except KafkaException as e:
+    print(f"Kafka error occurred: {e}")
+```
+
+## Performance Tips
+
+1. Batch messages when possible (`batch.num.messages` config)
+2. Adjust `linger.ms` for better batching
+3. Use `compression.type` (lz4, snappy, or gzip)
+4. Tune `fetch.max.bytes` and `max.partition.fetch.bytes` for consumers
+
+## License
+
+MIT

dgkafka-1.0.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+dgkafka/__init__.py,sha256=fnqVZROyHXipdmhqZaa9XUjvQe795JJKFakwTndAiIw,286
+dgkafka/avro_consumer.py,sha256=fI-VYQdw3bQJoDN4jS0F5Aa78wlxmjg5lmnTumw9FDs,2917
+dgkafka/avro_producer.py,sha256=UL0RBaEKl_lbNF1UI-4w8R3AIEGHYI-Vna3iVZIlJqo,5421
+dgkafka/config.py,sha256=TmfB0IOauSPr4B8InOZdoebAEGeabs7HZUXuozAFFlk,3828
+dgkafka/consumer.py,sha256=OfONH61xj1sD4Z0YIQngfJOOwkx8Ko9waNYCKoXtJL4,8371
+dgkafka/errors.py,sha256=PaH46tXameS--hrrUXKhQkZlBHvMSMPmjhVeRkmFvV0,95
+dgkafka/json_consumer.py,sha256=7Gzn7C9WpyCTPDV6eFDugAx5gC9vdV-HrTh3Nv--zIE,1152
+dgkafka/producer.py,sha256=WHVhw5HyJINxvUg_j_9iiQ0N7I3jc4niFXpZfkDHUOQ,6519
+dgkafka-1.0.0.dist-info/licenses/LICENSE,sha256=pAZXnNE2dxxwXFIduGyn1gpvPefJtUYOYZOi3yeGG94,1068
+dgkafka-1.0.0.dist-info/METADATA,sha256=QU2tM_MML3lTLbAjuBPpUgOgVOsvQMC8Fw_2AYy-FZ4,6059
+dgkafka-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dgkafka-1.0.0.dist-info/top_level.txt,sha256=GyNrxOh7IPdL0t2SxH8DWxg3fUma-ezQ1Kz4zIr2B7U,8
+dgkafka-1.0.0.dist-info/RECORD,,

dgkafka-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dgkafka-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dgkafka-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dgkafka