dgkafka 1.0.0a0__tar.gz → 1.0.0a2__tar.gz

{dgkafka-1.0.0a0 → dgkafka-1.0.0a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dgkafka
-Version: 1.0.0a0
+Version: 1.0.0a2
 Summary: Kafka clients
 Home-page: https://gitlab.com/gng-group/dgkafka.git
 Author: Malanris
@@ -8,17 +8,32 @@ Author-email: Roman Rasputin <admin@roro.su>
 License: MIT License
 Project-URL: Homepage, https://gitlab.com/gng-group/dgkafka
 Project-URL: BugTracker, https://gitlab.com/gng-group/dgkafka/issues
-Keywords: logging,logger,rotation,timed,advanced
+Keywords: kafka,client,confluent,avro,fastapi,logging
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Logging
-Requires-Python: >=3.7
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: confluent-kafka>=2.1.1
+Requires-Dist: dglog>=1.0.0
+Provides-Extra: avro
+Requires-Dist: requests; extra == "avro"
+Requires-Dist: fastavro<2; extra == "avro"
+Requires-Dist: avro<2,>=1.11.1; extra == "avro"
+Requires-Dist: attrs; extra == "avro"
+Requires-Dist: cachetools; extra == "avro"
+Requires-Dist: httpx>=0.26; extra == "avro"
+Requires-Dist: authlib; extra == "avro"
+Provides-Extra: json
+Requires-Dist: pyrsistent; extra == "json"
+Requires-Dist: jsonschema; extra == "json"
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file
@@ -39,6 +54,12 @@ For Avro support (requires additional dependencies):
 pip install dgkafka[avro]
 ```
 
+For Json support (requires additional dependencies):
+
+```bash
+pip install dgkafka[json]
+```
+
 ## Features
 
 - Producers and consumers for different data formats:

{dgkafka-1.0.0a0 → dgkafka-1.0.0a2}/README.md

@@ -14,6 +14,12 @@ For Avro support (requires additional dependencies):
 pip install dgkafka[avro]
 ```
 
+For Json support (requires additional dependencies):
+
+```bash
+pip install dgkafka[json]
+```
+
 ## Features
 
 - Producers and consumers for different data formats:

dgkafka-1.0.0a2/dgkafka/__init__.py

@@ -0,0 +1,11 @@
+from .consumer import KafkaConsumer
+from .producer import KafkaProducer
+try:
+    from .avro_consumer import AvroKafkaConsumer
+    from .avro_producer import AvroKafkaProducer
+except ImportError:
+    pass
+try:
+    from .json_consumer import JsonKafkaConsumer
+except ImportError:
+    pass

{dgkafka-1.0.0a0 → dgkafka-1.0.0a2}/dgkafka/avro_consumer.py

@@ -1,17 +1,20 @@
 from typing import Any, Iterator
 
 from dgkafka.consumer import KafkaConsumer
-from dglog import Logger
+
 from confluent_kafka import Message
 from confluent_kafka.avro import AvroConsumer
 from confluent_kafka.avro.serializer import SerializerError
 from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient
 
+import logging
+import dglog
+
 
 class AvroKafkaConsumer(KafkaConsumer):
     """Kafka consumer with Avro schema support using Schema Registry."""
 
-    def __init__(self, schema_registry_url: str, logger_: Logger | None = None, **configs: Any) -> None:
+    def __init__(self, logger_: logging.Logger | dglog.Logger | None = None, **configs: Any) -> None:
         """
         Initialize Avro consumer.
 
@@ -20,8 +23,8 @@ class AvroKafkaConsumer(KafkaConsumer):
             logger_: Optional logger instance
             configs: Kafka consumer configuration
         """
-        self.schema_registry_url = schema_registry_url
-        self.schema_registry_client = CachedSchemaRegistryClient({'url': schema_registry_url})
+        self.schema_registry_url = configs.get('schema.registry.url')
+        self.schema_registry_client = CachedSchemaRegistryClient({'url': self.schema_registry_url})
         super().__init__(logger_=logger_, **configs)
 
     def _init_consumer(self, **configs: Any) -> None:
@@ -35,7 +38,7 @@ class AvroKafkaConsumer(KafkaConsumer):
             self.logger.error(f"[x] Failed to initialize Avro consumer: {ex}")
             raise
 
-    def consume(self, num_messages: int = 1, timeout: float = 10.0, **kwargs: Any) -> Iterator[dict[str, Any] | None, Message]:
+    def consume(self, num_messages: int = 1, timeout: float = 10.0, **kwargs: Any) -> Iterator[str | bytes | Message | None]:
         """
         Consume Avro-encoded messages.
 
@@ -50,8 +53,9 @@ class AvroKafkaConsumer(KafkaConsumer):
         consumer = self._ensure_consumer()
 
         for _ in range(num_messages):
+            msg = self._consume(consumer, timeout)
             try:
-                if (msg := self._consume(consumer, timeout)) is None:
+                if msg is None:
                     continue
                 yield msg.value()
             except SerializerError as e:

{dgkafka-1.0.0a0 → dgkafka-1.0.0a2}/dgkafka/avro_producer.py

@@ -3,7 +3,8 @@ from confluent_kafka.avro import AvroProducer
 from confluent_kafka.avro.serializer import SerializerError
 from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient
 
-from dglog import Logger
+import dglog
+import logging
 
 from dgkafka.producer import KafkaProducer
 
@@ -13,10 +14,9 @@ class AvroKafkaProducer(KafkaProducer):
 
     def __init__(
             self,
-            schema_registry_url: str,
-            default_key_schema: Optional[Dict[str, Any]] = None,
-            default_value_schema: Optional[Dict[str, Any]] = None,
-            logger_: Optional[Logger] = None,
+            default_key_schema: str | None = None,
+            default_value_schema: str | None = None,
+            logger_: logging.Logger | dglog.Logger | None = None,
             **configs: Any
     ) -> None:
         """
@@ -29,17 +29,17 @@ class AvroKafkaProducer(KafkaProducer):
             logger_: Optional logger instance
             configs: Kafka producer configuration
         """
-        self.schema_registry_url = schema_registry_url
+        self.schema_registry_url = configs.get('schema.registry.url')
+        assert self.schema_registry_url is not None, "schema.registry.url is required"
+
         self.default_key_schema = default_key_schema
         self.default_value_schema = default_value_schema
-        self.schema_registry_client = CachedSchemaRegistryClient({'url': schema_registry_url})
+        self.schema_registry_client = CachedSchemaRegistryClient({'url': self.schema_registry_url})
         super().__init__(logger_=logger_, **configs)
 
     def _init_producer(self, **configs: Any) -> None:
         """Initialize AvroProducer instance."""
         try:
-            # AvroProducer requires schema registry in config
-            configs['schema.registry.url'] = self.schema_registry_url
             self.producer = AvroProducer(
                 config=configs,
                 default_key_schema=self.default_key_schema,
@@ -53,12 +53,12 @@ class AvroKafkaProducer(KafkaProducer):
     def produce(
             self,
             topic: str,
-            value: Union[Dict[str, Any], Any],
-            key: Optional[Union[Dict[str, Any], str]] = None,
-            value_schema: Optional[Dict[str, Any]] = None,
-            key_schema: Optional[Dict[str, Any]] = None,
-            partition: Optional[int] = None,
-            headers: Optional[Dict[str, bytes]] = None,
+            value: dict[str, Any] | Any,
+            key: dict[str, Any] | str | None  = None,
+            value_schema: dict[str, Any] | None = None,
+            key_schema: dict[str, Any] | None = None,
+            partition: int | None = None,
+            headers: dict[str, bytes] | None = None,
             flush: bool = True
     ) -> None:
         """

dgkafka-1.0.0a2/dgkafka/config.py

@@ -0,0 +1,89 @@
+from typing import Literal, Optional
+from pydantic import BaseModel, Field
+
+
+class KafkaConfig(BaseModel):
+    bootstrap_servers: str = Field(..., alias="bootstrap.servers",
+                                 description="Comma-separated list of broker addresses")
+    security_protocol: Optional[str] = Field(default="SSL", alias="security.protocol",
+                                           description="Protocol used to communicate with brokers")
+    ssl_ca_location: Optional[str] = Field(default=None, alias="ssl.ca.location",
+                                         description="Path to CA certificate file")
+    ssl_certificate_location: Optional[str] = Field(default=None, alias="ssl.certificate.location",
+                                                  description="Path to client certificate file")
+    ssl_key_location: Optional[str] = Field(default=None, alias="ssl.key.location",
+                                          description="Path to client private key file")
+    ssl_certificate_verification: Optional[bool] = Field(default=True,
+                                                       alias="enable.ssl.certificate.verification",
+                                                       description="Enable SSL certificate verification")
+    ssl_endpoint_identification_algorithm: Optional[str] = Field(default="https",
+                                                               alias="ssl.endpoint.identification.algorithm",
+                                                               description="Endpoint identification algorithm")
+
+    class Config:
+        allow_population_by_field_name = True
+        extra = "forbid"
+
+
+class ConsumerConfig(KafkaConfig):
+    group_id: str = Field(..., alias="group.id",
+                         description="Consumer group identifier")
+    enable_auto_commit: bool = Field(default=False, alias="enable.auto.commit",
+                                   description="Automatically commit offsets periodically")
+    auto_offset_reset: Literal["earliest", "latest"] = Field(default="earliest",
+                                                           alias="auto.offset.reset",
+                                                           description="Reset policy when no offset is available")
+    max_poll_records: Optional[int] = Field(default=500, alias="max.poll.records",
+                                          description="Maximum records per poll")
+    session_timeout_ms: int = Field(default=10000, alias="session.timeout.ms",
+                                  description="Timeout for consumer session")
+
+
+class ProducerConfig(KafkaConfig):
+    acks: Literal["all", "-1", "0", "1"] = Field(default="all",
+                                                description="Number of acknowledgments")
+    retries: int = Field(default=3, description="Number of retries on failure")
+    linger_ms: int = Field(default=0, alias="linger.ms",
+                         description="Delay in milliseconds to wait for messages in the buffer")
+    compression_type: Optional[str] = Field(default=None, alias="compression.type",
+                                          description="Compression codec to use")
+    batch_size: int = Field(default=16384, alias="batch.size",
+                          description="Batch size in bytes")
+
+
+class AvroConfigMixin(BaseModel):
+    schema_registry_url: str = Field(..., alias="schema.registry.url",
+                                   description="URL of Schema Registry")
+    schema_registry_ssl_ca_location: Optional[str] = Field(default=None,
+                                                         alias="schema.registry.ssl.ca.location",
+                                                         description="Schema Registry CA certificate path")
+    auto_register_schemas: bool = Field(default=True, alias="auto.register.schemas",
+                                      description="Automatically register schemas")
+
+
+class AvroConsumerConfig(ConsumerConfig, AvroConfigMixin):
+    use_latest_version: bool = Field(default=True, alias="use.latest.version",
+                                   description="Use latest schema version")
+    skip_known_types: bool = Field(default=False, alias="skip.known.types",
+                                 description="Skip known types during deserialization")
+
+
+class AvroProducerConfig(ProducerConfig, AvroConfigMixin):
+    value_subject_name_strategy: Optional[str] = Field(default=None,
+                                                     alias="value.subject.name.strategy",
+                                                     description="Strategy for subject name generation")
+    key_subject_name_strategy: Optional[str] = Field(default=None,
+                                                   alias="key.subject.name.strategy",
+                                                   description="Strategy for key subject name generation")
+
+
+class JsonConsumerConfig(ConsumerConfig):
+    json_deserializer: Optional[str] = Field(default=None,
+                                           description="Custom JSON deserializer function")
+    encoding: str = Field(default="utf-8", description="Message encoding")
+
+
+class JsonProducerConfig(ProducerConfig):
+    json_serializer: Optional[str] = Field(default=None,
+                                         description="Custom JSON serializer function")
+    encoding: str = Field(default="utf-8", description="Message encoding")

{dgkafka-1.0.0a0 → dgkafka-1.0.0a2}/dgkafka/consumer.py

@@ -1,17 +1,21 @@
 from typing import Literal, Iterator, Any
 from dgkafka.errors import ConsumerNotSetError
-from dglog import Logger
-from confluent_kafka import Consumer, KafkaError, KafkaException, Message, TopicPartition
+
+from confluent_kafka import Consumer, KafkaException, Message, TopicPartition
 from confluent_kafka import OFFSET_STORED, OFFSET_BEGINNING, OFFSET_END
 
+import logging
+import dglog
+
 OffsetType = Literal[OFFSET_STORED, OFFSET_BEGINNING, OFFSET_END] | int
 
 
 class KafkaConsumer:
-    def __init__(self, logger_: Logger | None = None, **configs: Any) -> None:
+    def __init__(self, logger_: logging.Logger | dglog.Logger | None = None, **configs: Any) -> None:
         self.consumer: Consumer | None = None
-        self.logger = logger_ if logger_ else Logger()
-        self.logger.auto_configure()
+        self.logger = logger_ if logger_ else dglog.Logger()
+        if isinstance(self.logger, dglog.Logger):
+            self.logger.auto_configure()
         self._init_consumer(**configs)
 
     def _init_consumer(self, **configs: Any) -> None:
@@ -84,10 +88,10 @@ class KafkaConsumer:
     def _consume(self, consumer: Consumer, timeout: float) -> Message | None:
         msg = consumer.poll(timeout)
         if msg is None:
-            return
+            return None
         if msg.error():
             self.logger.error(f"Consumer error: {msg.error()}")
-            return
+            return None
         self.logger.info(f"[<] Received message from {msg.topic()} [partition {msg.partition()}, offset {msg.offset()}]")
         self.logger.debug(f"[*] Message content: {msg.value()}")
         return msg

{dgkafka-1.0.0a0 → dgkafka-1.0.0a2}/dgkafka/producer.py

@@ -1,12 +1,15 @@
 import uuid
 from typing import Optional, Any
-from dglog import Logger
+
 from confluent_kafka import Producer, Message
 from dgkafka.errors import ProducerNotSetError
 
+import logging
+import dglog
+
 
 class KafkaProducer:
-    def __init__(self, logger_: Logger | None = None, **configs: Any) -> None:
+    def __init__(self, logger_: logging.Logger | dglog.Logger | None = None, **configs: Any) -> None:
         """Initialize Kafka producer.
 
         Args:
@@ -14,8 +17,9 @@ class KafkaProducer:
             configs: Kafka producer configuration
         """
         self.producer: Producer | None = None
-        self.logger = logger_ if logger_ else Logger()
-        self.logger.auto_configure()
+        self.logger = logger_ if logger_ else dglog.Logger()
+        if isinstance(self.logger, dglog.Logger):
+            self.logger.auto_configure()
         self._init_producer(**configs)
 
     def _init_producer(self, **configs: Any) -> None:

{dgkafka-1.0.0a0 → dgkafka-1.0.0a2}/dgkafka.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dgkafka
-Version: 1.0.0a0
+Version: 1.0.0a2
 Summary: Kafka clients
 Home-page: https://gitlab.com/gng-group/dgkafka.git
 Author: Malanris
@@ -8,17 +8,32 @@ Author-email: Roman Rasputin <admin@roro.su>
 License: MIT License
 Project-URL: Homepage, https://gitlab.com/gng-group/dgkafka
 Project-URL: BugTracker, https://gitlab.com/gng-group/dgkafka/issues
-Keywords: logging,logger,rotation,timed,advanced
+Keywords: kafka,client,confluent,avro,fastapi,logging
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Logging
-Requires-Python: >=3.7
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: confluent-kafka>=2.1.1
+Requires-Dist: dglog>=1.0.0
+Provides-Extra: avro
+Requires-Dist: requests; extra == "avro"
+Requires-Dist: fastavro<2; extra == "avro"
+Requires-Dist: avro<2,>=1.11.1; extra == "avro"
+Requires-Dist: attrs; extra == "avro"
+Requires-Dist: cachetools; extra == "avro"
+Requires-Dist: httpx>=0.26; extra == "avro"
+Requires-Dist: authlib; extra == "avro"
+Provides-Extra: json
+Requires-Dist: pyrsistent; extra == "json"
+Requires-Dist: jsonschema; extra == "json"
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file
@@ -39,6 +54,12 @@ For Avro support (requires additional dependencies):
 pip install dgkafka[avro]
 ```
 
+For Json support (requires additional dependencies):
+
+```bash
+pip install dgkafka[json]
+```
+
 ## Features
 
 - Producers and consumers for different data formats:

{dgkafka-1.0.0a0 → dgkafka-1.0.0a2}/dgkafka.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ setup.py
 dgkafka/__init__.py
 dgkafka/avro_consumer.py
 dgkafka/avro_producer.py
+dgkafka/config.py
 dgkafka/consumer.py
 dgkafka/errors.py
 dgkafka/json_consumer.py
@@ -13,4 +14,5 @@ dgkafka.egg-info/PKG-INFO
 dgkafka.egg-info/SOURCES.txt
 dgkafka.egg-info/dependency_links.txt
 dgkafka.egg-info/not-zip-safe
+dgkafka.egg-info/requires.txt
 dgkafka.egg-info/top_level.txt

dgkafka-1.0.0a2/dgkafka.egg-info/requires.txt

@@ -0,0 +1,15 @@
+confluent-kafka>=2.1.1
+dglog>=1.0.0
+
+[avro]
+requests
+fastavro<2
+avro<2,>=1.11.1
+attrs
+cachetools
+httpx>=0.26
+authlib
+
+[json]
+pyrsistent
+jsonschema

dgkafka-1.0.0a2/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dgkafka"
+version = "1.0.0a2"
+authors = [
+    {name = "Roman Rasputin", email = "admin@roro.su"},
+]
+description = "Kafka clients"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Operating System :: OS Independent",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: System :: Logging",
+    "Topic :: System :: Distributed Computing",
+]
+keywords = ["kafka", "client", "confluent", "avro", "fastapi", "logging"]
+license = {text = "MIT License"}
+dependencies = [
+    "confluent-kafka>=2.1.1",
+    "dglog>=1.0.0",
+]
+
+[project.optional-dependencies]
+avro = [
+    "requests",
+    "fastavro<2",
+    "avro>=1.11.1,<2",
+    "attrs",
+    "cachetools",
+    "httpx>=0.26",
+    "authlib"
+]
+json = [
+    "pyrsistent",
+    "jsonschema"
+]
+
+[project.urls]
+Homepage = "https://gitlab.com/gng-group/dgkafka"
+BugTracker = "https://gitlab.com/gng-group/dgkafka/issues"

dgkafka-1.0.0a0/dgkafka/__init__.py

@@ -1,5 +0,0 @@
-from consumer import KafkaConsumer
-from avro_consumer import AvroKafkaConsumer
-from producer import KafkaProducer
-from avro_producer import AvroKafkaProducer
-from json_consumer import JsonKafkaConsumer

dgkafka-1.0.0a0/pyproject.toml

@@ -1,28 +0,0 @@
-[build-system]
-requires = ["setuptools>=42", "wheel"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "dgkafka"
-version = "1.0.0a0"
-authors = [
-    {name="Roman Rasputin", email="admin@roro.su"},
-]
-description = "Kafka clients"
-readme = "README.md"
-requires-python = ">=3.7"
-classifiers = [
-    "Development Status :: 4 - Beta",
-    "Intended Audience :: Developers",
-    "License :: OSI Approved :: MIT License",
-    "Programming Language :: Python :: 3",
-    "Operating System :: OS Independent",
-    "Topic :: Software Development :: Libraries :: Python Modules",
-    "Topic :: System :: Logging",
-]
-keywords = ["logging", "logger", "rotation", "timed", "advanced"]
-license = {text = "MIT License"}
-
-[project.urls]
-Homepage = "https://gitlab.com/gng-group/dgkafka"
-BugTracker = "https://gitlab.com/gng-group/dgkafka/issues"