kafi 0.1.0__py3-none-any.whl

kafi/addons.py

@@ -0,0 +1,276 @@
+import json
+
+from kafi.functional import Functional
+from kafi.helpers import copy_kwargs
+
+# Constants
+
+ALL_MESSAGES = -1
+
+#
+
+def default_projection_function(message_dict1, message_dict2):
+    message_dict = dict(message_dict1)
+    message_dict["value"] = message_dict1["value"] | message_dict2["value"]
+    return message_dict
+#
+
+class AddOns(Functional):
+    def compact(self, topic, n=ALL_MESSAGES, **kwargs):
+        def foldl_function(acc, message_dict):
+            key_hash_int_message_dict_dict = acc
+            #
+            key = message_dict["key"]
+            value = message_dict["value"]
+            #
+            if key is not None:
+                key_hash_int = hash(str(key))
+                if value is None:
+                    if key_hash_int in key_hash_int_message_dict_dict:
+                        del key_hash_int_message_dict_dict[key_hash_int]
+                else:
+                    key_hash_int_message_dict_dict[key_hash_int] = message_dict
+            #
+            return key_hash_int_message_dict_dict
+        #
+
+        (key_hash_int_message_dict_dict, _) = self.foldl(topic, foldl_function, {}, n, **kwargs)
+        #
+        message_dict_list = list(key_hash_int_message_dict_dict.values())
+        #
+        return message_dict_list
+
+    def compact_to(self, topic, target_storage, target_topic, n=ALL_MESSAGES, **kwargs):
+        source_kwargs = copy_kwargs("source", **kwargs)
+        target_kwargs = copy_kwargs("target", **kwargs)
+        #
+        message_dict_list = self.compact(topic, n, **source_kwargs)
+        #
+        target_producer = target_storage.producer(target_topic, **target_kwargs)
+        key_bytes_list_value_bytes_list_tuple = target_producer.produce_list(message_dict_list, **target_kwargs)
+        target_producer.close()
+        #
+        return key_bytes_list_value_bytes_list_tuple
+
+    #
+
+    def join_to(self, source_topic1, source_storage2, source_topic2, target_storage, target_topic, get_key_function1=lambda x: x["key"], get_key_function2=lambda x: x["key"], projection_function=default_projection_function, join="left", n=ALL_MESSAGES, **kwargs):
+        join_str = join
+        #
+        if join_str not in ["inner", "left", "right"]:
+            raise Exception("Only \"inner\", \"left\" and \"right\" supported.")
+        #
+        def zip_foldl_to_function(acc, message_dict1, message_dict2):
+            # print(message_dict1["value"])
+            # print(message_dict2["value"])
+            # print("===")
+            (index_dict1, index_dict2) = acc
+            #
+            key1 = get_key_function1(message_dict1)
+            key2 = get_key_function2(message_dict2)
+            # DBSP: L join R = deltaL join deltaR + deltaL join R + L join deltaR
+            out_message_dict_list = []
+            # 1. deltaL join deltaR
+            if key1 == key2:
+                # Match in deltaL join deltaR.
+                out_message_dict_list.append(projection_function(message_dict1, message_dict2))
+            else:
+                # 2. deltaL join R
+                if key1 in index_dict2:
+                    # Match in deltaL join R.
+                    out_message_dict_list.append(projection_function(message_dict1, index_dict2[key1]))
+                else:
+                    # Could not find key1 in index_dict2.
+                    # Only append to the output if the join type is "left"
+                    if join_str == "left":
+                        out_message_dict_list.append(message_dict1)
+                # 3. L join deltaR
+                if key2 in index_dict1:
+                    # Match in L join deltaR
+                    out_message_dict_list.append(projection_function(index_dict1[key2], message_dict2))
+                else:
+                    # Could not find key2 in index_dict1.
+                    # Only append to the output if the join type is "right"
+                    if join_str == "right":
+                        out_message_dict_list.append(message_dict2)
+            # Depending on the join type, persist:
+            # * both sides (inner)
+            # * the left side (left)
+            # * the right side (right)
+            if join_str == "inner":
+                index_dict1[key1] = message_dict1
+                index_dict2[key2] = message_dict2
+            elif join_str == "left":
+                index_dict1[key1] = message_dict1
+            elif join_str == "right":
+                index_dict1[key2] = message_dict2
+            #
+            return ((index_dict1, index_dict2), list(out_message_dict_list))
+
+        return self.zip_foldl_to(source_topic1, source_storage2, source_topic2, target_storage, target_topic, zip_foldl_to_function, ({}, {}), n=n, **kwargs)
+
+    #
+
+    def repeat(self, topic_str, n=1, **kwargs):
+        n_int = n
+        #
+        message_dict_list = self.tail(topic_str, type="bytes", n=n_int, **kwargs)
+        pr = self.producer(topic_str, type="bytes", **kwargs)
+        pr.produce_list(message_dict_list, **kwargs)
+        pr.close()
+        #
+        return message_dict_list
+
+    #
+
+    def recreate(self, pattern, partitions=None, config={}, **kwargs):
+        pattern_str_or_str_list = pattern
+        #
+        topic_str_list = self.admin.list_topics(pattern_str_or_str_list)
+        #
+        if topic_str_list:
+            for topic_str in topic_str_list:
+                if partitions is None:
+                    partitions_int = self.partitions(topic_str)[topic_str]
+                else:
+                    partitions_int = partitions
+                #
+                old_config_dict = self.config(topic_str)[topic_str]
+                config_dict = {}
+                for key_str, value_str in old_config_dict.items():
+                    if key_str in config:
+                        config_dict[key_str] = config[key_str]
+                    else:
+                        config_dict[key_str] = value_str
+                #
+                self.delete(topic_str)
+                #
+                self.create(topic_str, partitions=partitions_int, config=config_dict, **kwargs)
+        else:
+            if isinstance(pattern_str_or_str_list, str):
+                topic_str_list = [pattern_str_or_str_list]
+            elif isinstance(pattern_str_or_str_list, list):
+                topic_str_list = pattern_str_or_str_list
+            #
+            for topic_str in topic_str_list:
+                if partitions is None:
+                    partitions_int = 1
+                else:
+                    partitions_int = partitions
+                #
+                self.create(topic_str, partitions=partitions_int, config=config, **kwargs)
+        #
+        return topic_str_list
+
+    retouch = recreate
+
+    #
+
+    def cp_group_offsets(self, pattern, source_group, target_storage, target_group):
+        source_group_str = source_group
+        target_group_str = target_group
+        #
+        topic_str_list = self.admin.list_topics(pattern)
+        #
+        # Get the offsets of the source consumer group.
+        topic_str_offsets_dict_dict = {topic_str: offsets_dict for topic_str, offsets_dict in self.group_offsets(source_group_str)[source_group_str].items() if topic_str in topic_str_list}
+        #
+        # Consume one message from eacg topic with the target consumer group to bring it to life.
+        for topic_str in topic_str_list:
+            co = target_storage.consumer(topic_str, group=target_group_str, type="bytes")
+            co.consume(n=1)
+            co.close()
+        #
+        target_group_offsets = target_storage.group_offsets(target_group, topic_str_offsets_dict_dict)
+        #
+        return target_group_offsets
+
+    #
+
+    def offsets_diff(self, pattern, ts, end_ts, **kwargs):
+        ts_int = ts
+        end_ts_int = end_ts
+        #
+        if end_ts_int < ts_int:
+            raise Exception(f"End timestamp ({end_ts_int}) before start timestamp ({ts_int}).")
+        #
+        topic_str_partitions_int_dict = self.partitions(pattern, **kwargs)
+        #
+        topic_str_messages_int_dict = {}
+        for topic_str, partitions_int in topic_str_partitions_int_dict.items():
+            start_offsets_dict = self.offsets_for_times(topic_str, {partition_int: ts_int for partition_int in range(partitions_int)}, replace_not_found=True, **kwargs)[topic_str]
+            end_offsets_dict = self.offsets_for_times(topic_str, {partition_int: end_ts_int for partition_int in range(partitions_int)}, replace_not_found=True, **kwargs)[topic_str]
+            #
+            # print(start_offsets_dict)
+            # print(end_offsets_dict)
+            #
+            messages_int = sum([(end_offset_int - start_offset_int) + 1 for start_offset_int, end_offset_int in zip(start_offsets_dict.values(), end_offsets_dict.values())])
+            #
+            topic_str_messages_int_dict[topic_str] = messages_int
+        #
+        return topic_str_messages_int_dict
+
+    #
+
+    def message_size(self, topic_str, **kwargs):
+        def agg(partition_int_offset_int_size_int_tuple_dict_dict, message_dict):
+            partition_int = message_dict["partition"]
+            offset_int = message_dict["offset"]
+            key_bytes = message_dict["key"]
+            key_size_int = 0 if key_bytes is None else len(key_bytes)
+            value_bytes = message_dict["value"]
+            value_size_int = 0 if value_bytes is None else len(value_bytes)
+            #
+            if partition_int not in partition_int_offset_int_size_int_tuple_dict_dict:
+                partition_int_offset_int_size_int_tuple_dict_dict[partition_int] = {offset_int: None}
+            partition_int_offset_int_size_int_tuple_dict_dict[partition_int][offset_int] = (key_size_int, value_size_int)
+            return partition_int_offset_int_size_int_tuple_dict_dict
+        #
+        (partition_int_offset_int_size_int_tuple_dict_dict, n_int) = self.foldl(topic_str, agg, {}, type="bytes", **kwargs)
+        #
+        return partition_int_offset_int_size_int_tuple_dict_dict, n_int
+    
+    def message_size_stats(self, topic_str, **kwargs):
+        partition_int_offset_int_size_int_tuple_dict_dict, n_int = self.message_size(topic_str, **kwargs)
+        #
+        total_size_int = 0
+        max_dict = {}
+        min_dict = {}
+        for partition_int, offset_int_size_int_tuple_dict in partition_int_offset_int_size_int_tuple_dict_dict.items():
+            for offset_int, (key_size_int, value_size_int) in offset_int_size_int_tuple_dict.items():
+                size_int = key_size_int + value_size_int
+                #
+                total_size_int += size_int
+                #
+                if max_dict == {}:
+                    max_dict = {"size": size_int, "partition": partition_int, "offset": offset_int}
+                else:
+                    old_max_int = max_dict["size"]
+                    new_max_int = max(size_int, old_max_int)
+                    if new_max_int != old_max_int:
+                        max_dict = {"size": new_max_int, "partition": partition_int, "offset": offset_int}
+                #
+                if min_dict == {}:
+                    min_dict = {"size": size_int, "partition": partition_int, "offset": offset_int}
+                else:
+                    old_min_int = min_dict["size"]
+                    new_min_int = min(size_int, old_min_int)
+                    if new_min_int != old_min_int:
+                        min_dict = {"size": new_min_int, "partition": partition_int, "offset": offset_int}
+                #
+        #
+        stats_dict = {"messages": n_int, "total_size": total_size_int, "average_size": total_size_int/n_int, "max_size": max_dict, "min_size": min_dict}
+        #
+        return stats_dict
+
+
+    def collect_value_set(self, topic_str, **kwargs):
+        value_json_str_set = set()
+        #
+        def collect(message_dict):
+            value_json_str = json.dumps(message_dict["value"])
+            value_json_str_set.add(value_json_str)
+        #
+        self.foreach(topic_str, foreach_function=collect, **kwargs)
+        #
+        return value_json_str_set

kafi/chunker.py

@@ -0,0 +1,63 @@
+import uuid
+
+from kafi.serializer import Serializer
+from kafi.helpers import message_dict_chunk_key_to_key, default_partitioner, key_to_chunk_key, split_bytes
+
+#
+
+class Chunker(Serializer):
+    def __init__(self, schema_registry_config_dict, **kwargs):
+        self.chunk_size_bytes_int = kwargs["chunk_size_bytes"] if "chunk_size_bytes" in kwargs else -1
+        if self.chunk_size_bytes_int == 0:
+            raise Exception("Chunk size is zero.")
+        if self.chunk_size_bytes_int > 0 and self.__class__.__name__ == "RestProxyProducer":
+            raise Exception("Chunking not supported for RestProxy storage.")
+        #
+        if self.chunk_size_bytes_int > 0:
+            self.partitioner_function = default_partitioner
+            self.projection_function = message_dict_chunk_key_to_key
+        #
+        super().__init__(schema_registry_config_dict, **kwargs)
+
+
+    #
+
+    def chunk(self, message_dict_list):
+        message_dict_list1 = []
+        if self.chunk_size_bytes_int > 0:
+            for message_dict in message_dict_list:
+                value_bytes = message_dict["value"]
+                #
+                if len(value_bytes) > self.chunk_size_bytes_int:
+                    chunk_value_bytes_list = split_bytes(value_bytes, self.chunk_size_bytes_int)
+                    #
+                    headers_str_bytes_tuple_list = message_dict["headers"]
+                    if headers_str_bytes_tuple_list is None:
+                        headers_str_bytes_tuple_list = []
+                    headers_str_bytes_dict = dict(headers_str_bytes_tuple_list)
+                    headers_str_bytes_dict["kafi_chunked_message_id"] = bytes(str(uuid.uuid4()), "UTF-8")
+                    headers_str_bytes_dict["kafi_number_of_chunks"] = len(chunk_value_bytes_list).to_bytes(32, byteorder="big")
+                    #
+                    for chunk_int, chunk_value_bytes in zip(range(len(chunk_value_bytes_list)), chunk_value_bytes_list):
+                        # If the first byte of the value starts with 0 we assume this is a message serialized using Schema Registry. In that case, add the five bytes from the beginning of the message to each chunk (to avoid confluent.value.schema.validation == true blocking the individual chunks).
+                        if self.value_type_str in ["avro", "jsonschema", "json_sr", "pb", "protobuf"] and chunk_int > 0:
+                            chunk_value_bytes = value_bytes[0:5] + chunk_value_bytes
+                        #
+                        chunk_key_bytes = key_to_chunk_key(message_dict["key"], chunk_int)
+                        #
+                        headers_str_bytes_dict["kafi_chunk_number"] = chunk_int.to_bytes(32, byteorder="big")
+                        #
+                        chunk_headers_str_bytes_tuple_list = list(headers_str_bytes_dict.items())
+                        #
+                        message_dict1 = {"value": chunk_value_bytes,
+                                         "key": chunk_key_bytes,
+                                         "partition": message_dict["partition"],
+                                         "timestamp": message_dict["timestamp"],
+                                         "headers": chunk_headers_str_bytes_tuple_list}
+                        message_dict_list1.append(message_dict1)
+                else:
+                    message_dict_list1 = message_dict_list
+        else:
+            message_dict_list1 = message_dict_list
+        #
+        return message_dict_list1

kafi/dechunker.py

@@ -0,0 +1,75 @@
+from kafi.deserializer import Deserializer
+from kafi.helpers import chunk_key_to_key
+
+#
+
+class Dechunker(Deserializer):
+    def __init__(self, schema_registry_config_dict, **kwargs):
+        # Dictionary mapping chunked message IDs to chunk numbers to chunk value_bytes (to reconstruct chunked messages).
+        self.chunks_dict = {}
+        #
+        super().__init__(schema_registry_config_dict, **kwargs)
+
+    #
+
+    def dechunk(self, message_dict_list):
+        message_dict_list1 = []
+        for message_dict in message_dict_list:
+            topic_str = message_dict["topic"]
+            # Get dictionary of headers
+            headers_str_bytes_tuple_list = message_dict["headers"]
+            if headers_str_bytes_tuple_list is None:
+                headers_str_bytes_tuple_list = []
+            headers_str_bytes_dict = dict(headers_str_bytes_tuple_list)
+            #
+            if "kafi_chunked_message_id" in headers_str_bytes_dict:
+                #
+                chunked_message_id_str = str(headers_str_bytes_dict["kafi_chunked_message_id"])
+                #
+                number_of_chunks_int = int.from_bytes(headers_str_bytes_dict["kafi_number_of_chunks"])
+                #
+                chunk_number_int = int.from_bytes(headers_str_bytes_dict["kafi_chunk_number"])
+                #
+                if chunked_message_id_str not in self.chunks_dict:
+                    self.chunks_dict[chunked_message_id_str] = {chunk_number_int1: None for chunk_number_int1 in range(number_of_chunks_int)}
+                #
+                self.chunks_dict[chunked_message_id_str][chunk_number_int] = message_dict["value"]
+                #
+                if all(value_bytes is not None for value_bytes in self.chunks_dict[chunked_message_id_str].values()):
+                    dechunked_value_bytes = b""
+                    #
+                    for chunk_number_int1, value_bytes in self.chunks_dict[chunked_message_id_str].items():
+                        # Special handling if the values were serialized in conjunction with Schema Registry.
+                        if self.topic_str_value_type_str_dict[topic_str] in ["avro", "jsonschema", "json_sr", "pb", "protobuf"]:
+                            # If so, skip the first five bytes from all but the first chunk (upon produce, we add the first five bytes to all messages to avoid confluent.value.schema.validation == true blocking them).
+                            if chunk_number_int1 == 0:
+                                dechunked_value_bytes += value_bytes
+                            else:
+                                dechunked_value_bytes += value_bytes[5:]
+                        # Else just dechunk.
+                        else:
+                            dechunked_value_bytes += value_bytes
+                    #
+                    key_bytes = chunk_key_to_key(message_dict["key"])
+                    #
+                    # Delete the header fields for chunking.
+                    del headers_str_bytes_dict["kafi_chunked_message_id"]
+                    del headers_str_bytes_dict["kafi_number_of_chunks"]
+                    del headers_str_bytes_dict["kafi_chunk_number"]
+                    headers_str_bytes_tuple_list = list(headers_str_bytes_dict.items())
+                    #
+                    message_dict2 = {"value": dechunked_value_bytes,
+                                        "key": key_bytes,
+                                        "headers": headers_str_bytes_tuple_list,
+                                        "timestamp": message_dict["timestamp"],
+                                        "partition": message_dict["partition"],
+                                        "offset": message_dict["offset"],
+                                        "topic": message_dict["topic"]}
+                    message_dict_list1.append(message_dict2)
+                    #
+                    # Clean up the chunks dictionary.
+                    del self.chunks_dict[chunked_message_id_str]
+            else:
+                message_dict_list1.append(message_dict)
+        #
+        return message_dict_list1

kafi/deserializer.py

@@ -0,0 +1,148 @@
+import os
+import importlib
+import json
+import sys
+import tempfile
+import uuid
+
+from confluent_kafka.schema_registry.avro import AvroDeserializer
+from confluent_kafka.schema_registry.json_schema import JSONDeserializer
+from confluent_kafka.schema_registry.protobuf import ProtobufDeserializer
+from confluent_kafka.serialization import MessageField, SerializationContext
+from google.protobuf.json_format import MessageToDict
+
+from kafi.schemaregistry import SchemaRegistry
+
+class Deserializer(SchemaRegistry):
+    def __init__(self, schema_registry_config_dict, **kwargs):
+        self.deser_from_dict = kwargs["deser_from_dict"] if "deser_from_dict" in kwargs else None
+        self.deser_conf = kwargs["deser_conf"] if "deser_conf" in kwargs else None
+        self.deser_rule_conf = kwargs["deser_rule_conf"] if "deser_rule_conf" in kwargs else None
+        self.deser_rule_registry = kwargs["deser_rule_registry"] if "deser_rule_registry" in kwargs else None
+        self.deser_json_decode = kwargs["deser_json_decode"] if "deser_json_decode" in kwargs else None
+        self.deser_return_record_name = kwargs["deser_return_record_name"] if "deser_return_record_name" in kwargs else False
+        #
+        super().__init__(schema_registry_config_dict)
+
+    def deserialize(self, payload_bytes, type_str, topic_str, headers_dict, key_bool):
+        if type_str.lower() == "bytes":
+            deserialized_payload = self.bytes_to_bytes(payload_bytes)
+        elif type_str.lower() in ["str", "string"]:
+            deserialized_payload = self.bytes_to_str(payload_bytes)
+        elif type_str.lower() == "json":
+            deserialized_payload = self.bytes_to_dict(payload_bytes)
+        elif type_str.lower() == "avro":
+            deserialized_payload = self.bytes_avro_to_dict(payload_bytes, topic_str, headers_dict, key_bool)
+        elif type_str.lower() in ["jsonschema", "json_sr"]:
+            deserialized_payload = self.bytes_jsonschema_to_dict(payload_bytes, topic_str, headers_dict, key_bool)
+        elif type_str.lower() in ["protobuf", "pb"]:
+            deserialized_payload = self.bytes_protobuf_to_dict(payload_bytes, topic_str, headers_dict, key_bool)
+        else:
+            raise Exception("Only \"str\", \"bytes\", \"json\", \"protobuf\" (\"pb\"), \"avro\" and \"jsonschema\" (\"json_sr\") supported.")
+        #
+        return deserialized_payload
+
+    def bytes_to_str(self, bytes):
+        if bytes:
+            return bytes.decode("utf-8")
+        else:
+            return bytes
+
+    def bytes_to_bytes(self, bytes):
+        return bytes
+
+    def bytes_to_dict(self, bytes):
+        if bytes is None:
+            return None
+        #
+        return json.loads(bytes)
+
+    def bytes_avro_to_dict(self, bytes, topic_str, headers_dict, key_bool):
+        if bytes is None:
+            return None
+        #
+        schema_str = self.get_schema_str(bytes, headers_dict, key_bool)
+        #
+        avroDeserializer = AvroDeserializer(self.schemaRegistryClient, schema_str, self.deser_from_dict, self.deser_return_record_name, self.deser_conf, self.deser_rule_conf, self.deser_rule_registry)
+        serializationContext = SerializationContext(topic_str, MessageField.KEY if key_bool else MessageField.VALUE)
+        dict = avroDeserializer(bytes, serializationContext)
+        return dict
+
+    def bytes_jsonschema_to_dict(self, bytes, topic_str, headers_dict, key_bool):
+        if bytes is None:
+            return None
+        #
+        schema_str = self.get_schema_str(bytes, headers_dict, key_bool)
+        #
+        jsonDeserializer = JSONDeserializer(schema_str, self.deser_from_dict, None, self.deser_conf, self.deser_rule_conf, self.deser_rule_registry, self.deser_json_decode)
+        serializationContext = SerializationContext(topic_str, MessageField.KEY if key_bool else MessageField.VALUE)
+        dict = jsonDeserializer(bytes, serializationContext)
+        return dict
+
+    def bytes_protobuf_to_dict(self, bytes, topic_str, headers_dict, key_bool):
+        if bytes is None:
+            return None
+        #
+        schema_id_int = int.from_bytes(bytes[1:5], "big")
+        if schema_id_int in self.schema_id_int_generalizedProtocolMessageType_protobuf_schema_str_tuple_dict:
+            generalizedProtocolMessageType, protobuf_schema_str = self.schema_id_int_generalizedProtocolMessageType_protobuf_schema_str_tuple_dict[schema_id_int]
+        else:
+            generalizedProtocolMessageType, protobuf_schema_str = self.schema_id_int_to_generalizedProtocolMessageType_protobuf_schema_str_tuple(schema_id_int)
+            self.schema_id_int_generalizedProtocolMessageType_protobuf_schema_str_tuple_dict[schema_id_int] = (generalizedProtocolMessageType, protobuf_schema_str)
+        #
+        # Prevent: RuntimeError: ProtobufSerializer: the 'use.deprecated.format' configuration property must be explicitly set due to backward incompatibility with older confluent-kafka-python Protobuf producers and consumers. See the release notes for more details
+        if self.deser_conf is None:
+            self.deser_conf = {"use.deprecated.format": False}
+        protobufDeserializer = ProtobufDeserializer(generalizedProtocolMessageType, self.deser_conf, None, self.deser_rule_conf, self.deser_rule_registry)
+        serializationContext = SerializationContext(topic_str, MessageField.KEY if key_bool else MessageField.VALUE)
+        protobuf_message = protobufDeserializer(bytes, serializationContext)
+        dict = MessageToDict(protobuf_message)
+        return dict
+
+    # Helpers
+
+    def get_schema_str(self, bytes, headers_dict, key_bool):
+        schema_id_key_str = "__key_schema_id" if key_bool else "__value_schema_id"
+        #
+        if headers_dict is not None and schema_id_key_str in headers_dict:
+            # Get the Schema ID from headers_dict if available.
+            schema_guid_bytes = headers_dict[schema_id_key_str]
+            # Skip the version byte (\x01).
+            schema_guid_bytes1 = schema_guid_bytes[1:]
+            # Convert to UUID.
+            schema_guid_str = str(uuid.UUID(bytes=schema_guid_bytes1))
+            # Get the schema at last.
+            schema_dict = self.get_schema_by_guid(schema_guid_str)
+        else:
+            # Else get the Schema ID from the payload.
+            schema_id_int = int.from_bytes(bytes[1:5], "big")
+            schema_dict = self.get_schema(schema_id_int)
+        #
+        schema_str = schema_dict["schema_str"]
+        #
+        return schema_str
+
+    def schema_id_int_to_generalizedProtocolMessageType_protobuf_schema_str_tuple(self, schema_id_int):
+        schema_dict = self.get_schema(schema_id_int)
+        schema_str = schema_dict["schema_str"]
+        #
+        generalizedProtocolMessageType = self.schema_id_int_and_schema_str_to_generalizedProtocolMessageType(schema_id_int, schema_str)
+        #
+        return generalizedProtocolMessageType, schema_str
+
+    def schema_id_int_and_schema_str_to_generalizedProtocolMessageType(self, schema_id_int, schema_str):
+        path_str = f"/{tempfile.gettempdir()}/kafi/protobuf/{self.storage_obj.config_str}"
+        os.makedirs(path_str, exist_ok=True)
+        file_str = f"schema_{schema_id_int}.proto"
+        file_path_str = f"{path_str}/{file_str}"
+        with open(file_path_str, "w") as textIOWrapper:
+            textIOWrapper.write(schema_str)
+        #
+        import grpc_tools.protoc
+        grpc_tools.protoc.main(["protoc", f"-I{path_str}", f"--python_out={path_str}", f"{file_str}"])
+        #
+        sys.path.insert(1, path_str)
+        schema_module = importlib.import_module(f"schema_{schema_id_int}_pb2")
+        schema_name_str = list(schema_module.DESCRIPTOR.message_types_by_name.keys())[0]
+        generalizedProtocolMessageType = getattr(schema_module, schema_name_str)
+        return generalizedProtocolMessageType

kafi/files.py

@@ -0,0 +1,85 @@
+import io
+import pathlib
+
+from kafi.pandas import Pandas
+
+# Constants
+
+ALL_MESSAGES = -1
+
+#
+
+#x = [{"name": "cookie", "calories": 500.0, "colour": "brown"}, {"name": "cake", "calories": 260.0, "colour": "white"}, {"name": "timtam", "calories": 80.0, "colour": "chocolate"}]
+
+class Files(Pandas):
+    def topic_to_file(self, topic, fs_obj, file, n=ALL_MESSAGES, **kwargs):
+        if not fs_obj.__class__.__bases__[0].__name__== "FS":
+            raise Exception("The target must be a file system.")
+        #
+        file_str = file
+        #
+        suffix_str = pathlib.Path(file_str).suffix
+        if suffix_str not in [".csv", ".json", ".parquet", ".xlsx", ".xml", ".bytes"]:
+            raise Exception("Only \".csv\", \".json\", \".parquet\", \".xlsx\", \".xml\" and \".bytes\" supported.")
+        #
+        if suffix_str == ".bytes":
+            message_dict_list = self.cat(topic, n, type="bytes", **kwargs)
+            data_bytes = b""
+            for message_dict in message_dict_list:
+                value_bytes = message_dict["value"]
+                data_bytes += value_bytes + b"\n"
+        else:
+            df = self.topic_to_df(topic, n, **kwargs)
+            data_bytesIO = io.BytesIO()
+            #
+            if suffix_str == ".csv":
+                index_bool = kwargs["index"] if "index" in kwargs else False
+                df.to_csv(data_bytesIO, index=index_bool)
+            elif suffix_str == ".json":
+                index_bool = kwargs["index"] if "index" in kwargs else None
+                df.to_json(data_bytesIO, orient="records")
+            elif suffix_str == ".parquet":
+                index_bool = kwargs["index"] if "index" in kwargs else None
+                df.to_parquet(data_bytesIO, index=index_bool, engine="fastparquet")
+            elif suffix_str == ".xlsx":
+                index_bool = kwargs["index"] if "index" in kwargs else False
+                df.to_excel(data_bytesIO, index=index_bool)
+            elif suffix_str == ".xml":
+                index_bool = kwargs["index"] if "index" in kwargs else False
+                df.to_xml(data_bytesIO, index=index_bool)
+            #
+            data_bytes = data_bytesIO.getvalue()
+        #
+        file_abs_path_str = fs_obj.admin.get_file_abs_path_str(file_str)
+        fs_obj.admin.write_bytes(file_abs_path_str, data_bytes)
+        #
+        return len(data_bytes)
+
+    def file_to_topic(self, file, storage_obj, topic, n=ALL_MESSAGES, **kwargs):
+        if not self.__class__.__bases__[0].__name__== "FS":
+            raise Exception("The source must be a file system.")
+        #
+        import pandas as pd
+        #
+        file_str = file
+        #
+        suffix_str = pathlib.Path(file_str).suffix
+        if suffix_str not in [".csv", ".json", ".parquet", ".xlsx", ".xml"]:
+            raise Exception("Only \".csv\", \".json\", \".parquet\", \".xlsx\" and \".xml\" supported.")
+        #
+        file_abs_path_str = self.admin.get_file_abs_path_str(file_str)
+        data_bytes = self.admin.read_bytes(file_abs_path_str)
+        data_bytesIO = io.BytesIO(data_bytes)
+        #
+        if suffix_str == ".csv":
+            df = pd.read_csv(data_bytesIO)
+        elif suffix_str == ".json":
+            df = pd.read_json(data_bytesIO)
+        elif suffix_str == ".parquet":
+            df = pd.read_parquet(data_bytesIO)
+        elif suffix_str == ".xlsx":
+            df = pd.read_excel(data_bytesIO)
+        elif suffix_str == ".xml":
+            df = pd.read_xml(data_bytesIO)
+        #
+        return storage_obj.df_to_topic(df, topic, n, **kwargs)