lcdp-kafka-utils 1.5.8__tar.gz

lcdp_kafka_utils-1.5.8/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: lcdp-kafka-utils
+Version: 1.5.8
+Summary: Kafka Utils
+Author: Le Comptoir Des Pharmacies
+Author-email: webmaster@lecomptoirdespharmacies.fr
+Requires-Python: >=3.8
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: confluent_kafka (==2.5.0)
+Requires-Dist: fastavro (==1.9.4)

lcdp_kafka_utils-1.5.8/lcdp_kafka_utils/batch_deserializing_consumer.py

@@ -0,0 +1,62 @@
+from confluent_kafka.cimpl import Consumer as _ConsumerImpl
+from confluent_kafka.error import (ConsumeError,
+                    KeyDeserializationError,
+                    ValueDeserializationError)
+from confluent_kafka.serialization import (SerializationContext,
+                            MessageField)
+
+
+class BatchDeserializingConsumer(_ConsumerImpl):
+
+    def __init__(self, conf):
+        conf_copy = conf.copy()
+        self._key_deserializer = conf_copy.pop('key.deserializer', None)
+        self._value_deserializer = conf_copy.pop('value.deserializer', None)
+        super(BatchDeserializingConsumer, self).__init__(conf_copy)
+
+    def consume(self, num_messages=1, timeout=-1):
+
+        # consume num_messages messages
+        records = super(BatchDeserializingConsumer, self).consume(num_messages, timeout)
+        # contain all deserialized records
+        deserialized_records = []
+
+        for record in records:
+            try:
+                if record is None:
+                    continue
+
+                # deserialize the record
+                deserialized_records.append(self.deserialize_record(record))
+
+            except Exception as e:
+                print("Error when deserializing record: {}".format(record), e)
+                continue
+
+        return deserialized_records
+
+    def deserialize_record(self, record):
+
+        if record.error() is not None:
+            raise ConsumeError(record.error(), kafka_message=record)
+
+        ctx = SerializationContext(record.topic(), MessageField.VALUE, record.headers())
+        value = record.value()
+        if self._value_deserializer is not None:
+            try:
+                value = self._value_deserializer(value, ctx)
+            except Exception as se:
+                raise ValueDeserializationError(exception=se, kafka_message=record)
+
+        key = record.key()
+        ctx.field = MessageField.KEY
+        if self._key_deserializer is not None:
+            try:
+                key = self._key_deserializer(key, ctx)
+            except Exception as se:
+                raise KeyDeserializationError(exception=se, kafka_message=record)
+
+        record.set_key(key)
+        record.set_value(value)
+
+        return record

lcdp_kafka_utils-1.5.8/lcdp_kafka_utils/cdc_dto_transformer.py

@@ -0,0 +1,160 @@
+import logging
+import threading
+import time
+
+from confluent_kafka.serializing_producer import SerializingProducer
+from .config import check_configuration, build_consumer_config, build_producer_config
+from .batch_deserializing_consumer import BatchDeserializingConsumer
+
+from confluent_kafka import KafkaException
+
+log = logging.getLogger(__name__)
+
+CONSUMER_POLL_TIMEOUT = 10.0
+DURATION_BEFORE_RESTART = 5.0
+N_BATCH_RECORDS = 1000
+
+
+def delivery_report(err, msg):
+    """
+    Reports the failure or success of a message delivery.
+    Args:
+        err (KafkaError): The error that occurred on None on success.
+        msg (Message): The message that was produced or failed.
+    Note:
+        In the delivery report callback the Message.key() and Message.value()
+        will be the binary format as encoded by any configured Serializers and
+        not the same object that was passed to produce().
+        If you wish to pass the original object(s) for key and value to delivery
+        report callback we recommend a bound callback or lambda where you pass
+        the objects along.
+    """
+    if err is not None:
+        log.error("Delivery failed for record {}: {}".format(msg.key(), err))
+        return
+    log.debug('Record {} successfully produced to {} [{}] at offset {}'.format(
+        msg.key(), msg.topic(), msg.partition(), msg.offset()))
+
+
+class CdcDtoTransformer(object):
+
+    def __init__(self, topic_in, topic_out):
+        self.topic_in = topic_in
+        self.topic_out = topic_out
+        self.running = False
+
+    def run(self):
+        if not check_configuration():
+            raise Exception(f"Missing environment variable to run Kafka")
+
+        if not self.topic_in or not self.topic_out:
+            raise Exception(f"Empty topic in/out for CDC to DTO transformer")
+
+        # RUNNING CONSUMER FOR READING MESSAGE FROM THE KAFKA TOPIC
+        t = threading.Thread(target=self.start_listener, name="Start kafka listener")
+        t.daemon = True
+        t.start()
+
+    def start_listener(self):
+        while True:
+            try:
+                self.__process_events()
+            except Exception as ex:
+                log.error("Exception occured in Kafka Events Processing Thread: {}", ex)
+                # pause a bit before restart
+                log.info("Going to restart in : {} seconds".format(DURATION_BEFORE_RESTART))
+                time.sleep(DURATION_BEFORE_RESTART)
+                pass
+
+    def __process_events(self):
+        try:
+            self.running = True
+            consumer = self.create_consumer()
+            producer = self.create_producer()
+
+            while self.running:
+                try:
+                    # read N_MESSAGE_BATCH messages at the same time
+                    records = consumer.consume(N_BATCH_RECORDS, CONSUMER_POLL_TIMEOUT)
+
+                except KafkaException as ex:
+                    # Ignore the exception like TOPIC_NOT_FOUND, PARTITTION_NOT_FOUND
+                    log.error("CdcDtoTransformer exception", ex)
+                    continue
+
+
+                for record in records:
+
+                    if record is None:
+                        # There is no new record in the topic, do nothing
+                        continue
+
+                    if record.error():
+                        print("Error reading message : {}".format(record.error()))
+                        continue
+
+                    if record.key() is None:
+                        log.error("MUST FIX - Strange null key processed with value : {}".format(record.value()))
+                        continue
+
+                    if record.value() is None:
+                        # Debezium by default generates a tombstone record to enable Kafka compaction after a delete record was generated.
+                        # This record is usually filtered out to avoid duplicates as a delete record is converted to a tombstone record, too
+                        continue
+
+                    # 'after' can be null if ressource deleted or in case of update with capture.mode <> change_streams_update_full (default since 1.8.0.Alpha1)
+                    # In this situation, just publish an event in the topic with null value
+                    cdc_value = record.value().get('after', None)
+                    cdc_key = record.key()
+
+                    dto_value = self.build_value(cdc_value)
+                    dto_key = self.build_key(cdc_key)
+
+                    producer.produce(topic=self.topic_out,
+                                     key=dto_key.dict() if dto_key else None,
+                                     value=dto_value.dict() if dto_value else None,
+                                     on_delivery=delivery_report)
+
+                # Flush all message and commit after produce batch records
+                producer.flush()
+                consumer.commit(asynchronous=True)
+        finally:
+            if consumer:
+                log.info("closing consumer")
+                consumer.close()
+
+            if producer:
+                log.info("Flushing records...")
+                producer.flush()
+
+    def build_key(self, msg):
+        raise NotImplementedError("Please Implement this method")
+
+    def build_value(self, msg):
+        raise NotImplementedError("Please Implement this method")
+
+    def get_key_schema(self):
+        raise NotImplementedError("Please Implement this method")
+
+    def get_value_schema(self):
+        raise NotImplementedError("Please Implement this method")
+
+    def get_name(self):
+        raise NotImplementedError("Please Implement this method")
+
+    def get_version(self):
+        raise NotImplementedError("Please Implement this method")
+
+    def create_producer(self):
+        config = build_producer_config(self.get_key_schema(), self.get_value_schema())
+        batch_producer = SerializingProducer(config)
+        return batch_producer
+
+    def create_consumer(self):
+        config = build_consumer_config(self.get_name(), self.get_version())
+        batch_consumer = BatchDeserializingConsumer(config)
+        batch_consumer.subscribe([self.topic_in])
+        return batch_consumer
+
+    def shutdown(self):
+        self.running = False

lcdp_kafka_utils-1.5.8/lcdp_kafka_utils/config.py

@@ -0,0 +1,89 @@
+import os
+from confluent_kafka.schema_registry import SchemaRegistryClient
+from confluent_kafka.schema_registry.avro import AvroDeserializer, AvroSerializer
+
+SERVICE_NAME = os.getenv('SERVICE_NAME')
+HERMES_BROKER = os.getenv('HERMES_BROKER')
+HERMES_USERNAME = os.getenv('HERMES_USERNAME')
+HERMES_PASSWORD = os.getenv('HERMES_PASSWORD')
+HERMES_SCHEMA_REGISTRY_URL = os.getenv('HERMES_SCHEMA_REGISTRY_URL')
+
+
+def check_configuration():
+    return HERMES_BROKER and HERMES_SCHEMA_REGISTRY_URL
+
+
+def build_schema_registry_config():
+    config = {
+        'url': HERMES_SCHEMA_REGISTRY_URL,
+    }
+
+    if HERMES_USERNAME:
+        config = {
+            **config,
+            **{
+                'basic.auth.user.info': "{}:{}".format(HERMES_USERNAME, HERMES_PASSWORD)
+            }
+        }
+
+    return config
+
+
+def build_base_config():
+    config = {
+        'bootstrap.servers': HERMES_BROKER,
+    }
+
+    if HERMES_USERNAME:
+        config = {
+            **config,
+            **{
+                'security.protocol': 'SASL_SSL',
+                'sasl.mechanism': 'SCRAM-SHA-512',
+                'sasl.username': HERMES_USERNAME,
+                'sasl.password': HERMES_PASSWORD,
+            }
+        }
+
+    return config
+
+
+def build_consumer_config(name, version):
+    schema_registry_client = SchemaRegistryClient(
+        build_schema_registry_config()
+    )
+    avro_deserializer = AvroDeserializer(schema_registry_client)
+
+    # Compute GROUP_ID
+    application_name = HERMES_USERNAME if HERMES_USERNAME else SERVICE_NAME
+    group_id = "{}.{}.{}".format(application_name, name, version)
+
+    return {
+        **build_base_config(),
+        **{
+            'group.id': group_id,
+            'auto.offset.reset': 'earliest',
+            'enable.auto.commit': 'false',
+            'key.deserializer': avro_deserializer,
+            'value.deserializer': avro_deserializer
+        }
+    }
+
+
+def build_producer_config(key_schema_str, value_schema_str):
+    schema_registry_client = SchemaRegistryClient(
+        build_schema_registry_config()
+    )
+    avro_serializer_configuration = {"auto.register.schemas": True}
+    key_serializer = AvroSerializer(schema_str=key_schema_str, schema_registry_client=schema_registry_client,
+                                    conf=avro_serializer_configuration)
+    value_serializer = AvroSerializer(schema_str=value_schema_str, schema_registry_client=schema_registry_client,
+                                      conf=avro_serializer_configuration)
+
+    return {
+        **build_base_config(),
+        **{
+            'key.serializer': key_serializer,
+            'value.serializer': value_serializer
+        }
+    }

lcdp_kafka_utils-1.5.8/lcdp_kafka_utils/dto_action_execute.py

@@ -0,0 +1,98 @@
+import logging
+import threading
+import time
+
+from confluent_kafka import KafkaException
+from .config import check_configuration, build_consumer_config
+from .batch_deserializing_consumer import BatchDeserializingConsumer
+
+log = logging.getLogger(__name__)
+
+CONSUMER_POLL_TIMEOUT = 10.0
+DURATION_BEFORE_RESTART = 5.0
+N_BATCH_RECORDS = 1000
+
+
+class DtoActionExecute(object):
+
+    def __init__(self, topic_in):
+        self.topic_in = topic_in
+        self.running = False
+
+    def run(self):
+        if not check_configuration():
+            raise Exception(f"Missing environment variable to run Kafka")
+
+        if not self.topic_in:
+            raise Exception(f"Empty topic in for DTO")
+
+        # RUNNING CONSUMER FOR READING MESSAGE FROM THE KAFKA TOPIC
+        t = threading.Thread(target=self.start_listener, name="Start kafka listener")
+        t.daemon = True
+        t.start()
+
+    def start_listener(self):
+        while True:
+            try:
+                self.__process_events()
+            except Exception as ex:
+                log.error("Exception occured in Kafka Events Processing Thread: {}", ex)
+                # pause a bit before restart
+                log.info("Going to restart in : {} seconds".format(DURATION_BEFORE_RESTART))
+                time.sleep(DURATION_BEFORE_RESTART)
+                pass
+
+    def __process_events(self):
+        try:
+            self.running = True
+            consumer = self.create_consumer()
+
+            while self.running:
+                try:
+                    # read N_MESSAGE_BATCH messages at the same time
+                    records = consumer.consume(N_BATCH_RECORDS, CONSUMER_POLL_TIMEOUT)
+
+
+                except KafkaException as ex:
+                    # Ignore the exception like TOPIC_NOT_FOUND, PARTITTION_NOT_FOUND
+                    log.error("DtoActionExecute exception", ex)
+                    continue
+
+                if records:
+                    self.action_events(records)
+
+                consumer.commit(asynchronous=True)
+
+        finally:
+            if consumer:
+                log.info("closing consumer")
+                consumer.close()
+
+    def action_events(self, records):
+        raise NotImplementedError("Please Implement this method")
+
+    def get_name(self):
+        raise NotImplementedError("Please Implement this method")
+
+    def get_version(self):
+        raise NotImplementedError("Please Implement this method")
+
+    def create_consumer(self):
+        config = build_consumer_config(self.get_name(), self.get_version())
+        batch_consumer = BatchDeserializingConsumer(config)
+        batch_consumer.subscribe([self.topic_in])
+        return batch_consumer
+
+    def shutdown(self):
+        self.running = False
+
+    @staticmethod
+    def dedupe(records):
+        unique_keys = []
+        deduped_records = []
+        for record in list(reversed(records)):
+            if record.key() not in unique_keys:
+                deduped_records.append(record)
+                unique_keys.append(record.key())
+
+        return list(reversed(deduped_records))

lcdp_kafka_utils-1.5.8/pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "lcdp-kafka-utils"
+# https://github.com/python-poetry/poetry/issues/1208
+version = "1.5.8"
+description = "Kafka Utils"
+authors = ["Le Comptoir Des Pharmacies <webmaster@lecomptoirdespharmacies.fr>"]
+
+[tool.poetry-dynamic-versioning]
+enable = false
+vcs = "git"
+
+[tool.poetry.requires-plugins]
+poetry-dynamic-versioning = { version = ">=1.0.0,<2.0.0", extras = ["plugin"] }
+
+[tool.poetry.dependencies]
+python = ">=3.8"
+confluent_kafka = "2.5.0"
+fastavro = "1.9.4"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
+build-backend = "poetry_dynamic_versioning.backend"