mx-stream-core 0.1.2__py3-none-any.whl

mx_stream_core/config/app.py

@@ -0,0 +1,4 @@
+import os
+
+app_name = os.getenv('APP_NAME', 'Mindx CDP')
+service_name = os.getenv('SERVICE_NAME', 'udp')

mx_stream_core/config/delta.py

@@ -0,0 +1,6 @@
+import os
+from mx_stream_core.config.s3 import s3_enable, s3_bucket
+
+delta_folder = os.getenv('DELTA_FOLDER', ".delta")
+default_delta_path = f'{s3_bucket}/{delta_folder}' if s3_enable else '/tmp/.delta'
+delta_path = os.getenv('DELTA_PATH', default_delta_path)

mx_stream_core/config/event_store.py

@@ -0,0 +1,3 @@
+import os
+
+eventstore_connection_string = os.getenv('EVENT_STORE_CONNECTION_STRING', 'esdb://localhost:2113?tls=false')

mx_stream_core/config/kafka.py

@@ -0,0 +1,3 @@
+import os
+
+kafka_bootstrap_servers = os.getenv('KAFKA_HOST', 'kafka:9092')

mx_stream_core/config/layer_events.py

@@ -0,0 +1,11 @@
+word = {
+    "ingested": 'INGESTED',
+}
+book = {
+    "ingested": 'INGESTED',
+}
+
+common = {
+    "ingested": 'INGESTED',
+    "transformed": 'TRANSFORMED',
+}

mx_stream_core/config/s3.py

@@ -0,0 +1,7 @@
+import os
+
+s3_access_key = os.getenv('S3_ACCESS_KEY', 'minio')
+s3_secret_key = os.getenv('S3_SECRET_KEY', 'minio123')
+s3_endpoint = os.getenv('S3_ENDPOINT', 'http://localhost:9000')
+s3_bucket = os.getenv('S3_BUCKET', 'cdp')
+s3_enable = os.getenv('S3_ENABLE', 'true')

mx_stream_core/config/spark.py

@@ -0,0 +1,3 @@
+import os
+
+master_url = os.getenv('MASTER_URL', "local[*]")

mx_stream_core/config/streams.py

@@ -0,0 +1,2 @@
+word = 'words'
+book = "books"

mx_stream_core/infrastructure/delta.py

@@ -0,0 +1,8 @@
+from mx_stream_core.config.delta import delta_path
+
+
+def get_delta_path(table_name=None) -> str:
+    if table_name is not None:
+        return f"{delta_path}/{table_name}"
+    print("path: ", delta_path)
+    return delta_path

mx_stream_core/infrastructure/event_store.py

@@ -0,0 +1,39 @@
+import logging
+import os
+from mx_stream_core.infrastructure.redis import get_stream_position
+from esdbclient import EventStoreDBClient
+
+"""
+Event Store client
+"""
+_client = None
+
+
+def get_stream(stream_name, group):
+    """
+    Get stream from event store
+    :param stream_name: event store stream name
+    :param group: event store group name this group will be used to get the stream position key from redis
+    :return:
+    """
+    stream_position = get_stream_position(stream_name, group)
+    logging.info(f"[Event Store] subscribe {stream_name} stream from position {stream_position}")
+
+    return get_event_store_client().subscribe_to_stream(
+        stream_name=stream_name,
+        stream_position=stream_position
+    )
+
+
+def get_event_store_client():
+    global _client
+    if _client is None:
+        """
+        Event Store connection string
+        """
+        eventstore_connection_string = os.getenv('EVENT_STORE_CONNECTION_STRING', 'esdb://localhost:2113?tls=false')
+
+        if eventstore_connection_string is None:
+            raise ValueError("EVENT_STORE_CONNECTION_STRING is required")
+        _client = EventStoreDBClient(eventstore_connection_string)
+    return _client

mx_stream_core/infrastructure/kafka.py

@@ -0,0 +1,139 @@
+from kafka import KafkaProducer
+from pyspark.sql import DataFrame
+from pyspark.sql.functions import col, from_json, to_json, struct
+from typing_extensions import deprecated
+
+from mx_stream_core.config.kafka import kafka_bootstrap_servers
+from mx_stream_core.config.app import service_name
+
+# Create Producer instance
+_producer = None
+
+
+def get_kafka_producer():
+    """
+    Get a Kafka producer
+    :return:
+    """
+    global _producer
+    if _producer is None:
+        _producer = KafkaProducer(
+            bootstrap_servers=kafka_bootstrap_servers,
+            value_serializer=lambda v: str(v).encode('utf-8')
+        )
+    return _producer
+
+
+def read_stream_from_kafka(spark, topic, schema) -> DataFrame:
+    """
+    Read a stream from Kafka
+    :param spark: SparkSession
+    :param topic: Kafka topic
+    :param schema: Schema of the data
+    :return:
+    """
+    df = spark \
+        .readStream \
+        .format("kafka") \
+        .option("kafka.bootstrap.servers", kafka_bootstrap_servers) \
+        .option("subscribe", topic) \
+        .option("startingOffsets", "earliest") \
+        .load()
+    df = df.selectExpr("CAST(value AS STRING) as json_str")
+    df = df.withColumn("json_data", from_json(col("json_str"), schema)).select("json_data.*")
+    return df
+
+
+def write_stream_to_kafka(df, topic, checkpoint_path):
+    """
+    Write a stream to Kafka
+    :param df: DataFrame
+    :param topic: Kafka topic
+    :param checkpoint_path: Checkpoint path for the stream
+    :return:
+    """
+    # Convert DataFrame to JSON strings
+    json = df.select(to_json(struct(
+        *df.columns
+    )).alias("value"))
+
+    # Write the merged stream to Kafka
+    query = json \
+        .writeStream \
+        .format("kafka") \
+        .option("kafka.bootstrap.servers", kafka_bootstrap_servers) \
+        .option("topic", topic) \
+        .option("checkpointLocation", checkpoint_path) \
+        .start()
+
+    return query
+
+
+def delivery_report(err, msg):
+    """
+    Reports the result of a message delivery attempt.
+    :param err: The error that occurred or None if the message was delivered successfully.
+    :param msg: The message that was sent or failed to send.
+    :return:
+    """
+    if err is not None:
+        print('Message delivery failed: {}'.format(err))
+    else:
+        print('Message delivered to {} [{}]'.format(msg.topic(), msg.partition()))
+
+
+def produce_kafka_message(topic, message):
+    """
+    Produce a message to Kafka
+    :param topic: Kafka topic
+    :param message:  Message to send
+    :return:
+    """
+    producer = get_kafka_producer()
+    producer.send(topic, value=message).add_errback(delivery_report)
+    producer.flush()  # Wait for all messages to be delivered
+
+
+def produce_kafka_messages(topic, event_name: str, df: DataFrame):
+    """
+    Produce a message to Kafka
+    :param topic: Kafka topic
+    :param message:  Message to send
+    :return:
+    """
+    producer = get_kafka_producer()
+    for row in df.toLocalIterator():
+        message = {
+            "event": event_name,
+            "data": row.asDict()
+        }
+        producer.send(topic, value=message).add_errback(delivery_report)
+    producer.flush()  # Wait for all messages to be delivered
+
+
+@deprecated("Use create_ingestor_topic_name or create_transformation_topic_name instead")
+def create_topic_name(ingestor_name: str) -> str:
+    """
+    Create a topic name
+    :param ingestor_name: Name of the ingestor (e.g. "word")
+    :return:
+    """
+    return f'{service_name}_ingestor_{ingestor_name}'
+
+
+def create_ingestor_topic_name(ingestor_name: str) -> str:
+    """
+    Create a topic name for an ingestor
+    :param ingestor_name: Name of the ingestor (e.g. "word")
+    :return:
+    """
+    return f'{service_name}_ingestor_{ingestor_name}'
+
+
+def create_transformation_topic_name(transformation_name: str) -> str:
+    """
+    Create a topic name for a transformation
+    :param transformation_name: Name of the transformation (e.g. "book")
+    :return:
+    """
+    return f'{service_name}_transformation_{transformation_name}'

mx_stream_core/infrastructure/redis.py

@@ -0,0 +1,63 @@
+import logging
+
+import redis
+import os
+
+from mx_stream_core.config.app import app_name
+
+redis_host = os.getenv('REDIS_HOST', 'localhost')
+redis_port = os.getenv('REDIS_PORT', 6379)
+redis_db = os.getenv('REDIS_DB', 0)
+redis_password = os.getenv('REDIS_PASSWORD', None)
+
+_r = None
+
+"""
+Log the Redis connection details
+"""
+logging.info(f'[Redis] host: {redis_host}, port: {redis_port}, db: {redis_db}')
+
+
+def get_redis_client():
+    """
+    Get a Redis client
+    :return:
+    """
+    global _r
+    if _r is None:
+        _r = redis.Redis(host=redis_host, port=redis_port, db=redis_db, password=redis_password)
+    return _r
+
+
+def get_stream_key(stream_name, group):
+    """
+    Get the key for the stream position
+    :param stream_name: event store stream name
+    :param group: stream group
+    :return:
+    """
+    return "{}_{}_{}_stream_position".format(app_name, group, stream_name)
+
+
+def get_stream_position(stream_name, group):
+    """
+    Get the stream position from Redis
+    :param stream_name: event store stream name
+    :param group: stream group
+    :return:
+    """
+    key = get_stream_key(stream_name, group)
+    stream_position = get_redis_client().get(key)
+    return int(stream_position) if stream_position else 0
+
+
+def set_stream_position(stream_name, group, position):
+    """
+    Set the stream position in Redis
+    :param stream_name:
+    :param group:
+    :param position:
+    :return:
+    """
+    key = get_stream_key(stream_name, group)
+    get_redis_client().set(key, position)

mx_stream_core/infrastructure/spark.py

@@ -0,0 +1,28 @@
+import os.path
+from pyspark.sql import SparkSession
+
+from mx_stream_core.config.app import app_name
+from mx_stream_core.config.s3 import s3_access_key, s3_secret_key, s3_endpoint, s3_enable, s3_bucket
+from mx_stream_core.config.spark import master_url
+
+spark_builder = SparkSession.builder.appName(app_name).master(master_url) \
+    .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \
+    .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+
+if s3_enable == 'true':
+    spark_builder.config("spark.hadoop.fs.s3a.access.key", s3_access_key) \
+        .config("spark.hadoop.fs.s3a.secret.key", s3_secret_key) \
+        .config("spark.hadoop.fs.s3a.endpoint", s3_endpoint)
+
+spark = spark_builder.getOrCreate()
+spark.sparkContext.setLogLevel(os.getenv('LOG_LEVEL', 'WARN'))
+
+checkpoint_folder = os.getenv('CHECKPOINT_FOLDER', ".checkpoints")
+default_root_checkpoint_path = f'{s3_bucket}/{checkpoint_folder}' if s3_enable else '/tmp/.checkpoints'
+root_check_point_path = os.getenv('CHECKPOINT_PATH', default_root_checkpoint_path)
+
+
+def get_checkpoint_path(table_name=None) -> str:
+    if table_name is not None:
+        return f"{root_check_point_path}/{table_name}_checkpoint"
+    return f"{root_check_point_path}/checkpoint"

mx_stream_core/utils/cmd.py

@@ -0,0 +1,9 @@
+import argparse
+
+
+def extract_args():
+    parser = argparse.ArgumentParser(description="Process some named parameters.")
+    parser.add_argument('--worker', type=str, required=False, help='Worker to run')
+    parser.add_argument('--layer', type=str, required=False, help='Layer to run')
+    args = parser.parse_args()
+    return args

mx_stream_core-0.1.2.dist-info/LICENSE.txt

@@ -0,0 +1 @@
+OK

mx_stream_core-0.1.2.dist-info/METADATA

@@ -0,0 +1,16 @@
+Metadata-Version: 2.1
+Name: mx_stream_core
+Version: 0.1.2
+Summary: This is package stream core of mindx
+Home-page: http://pypi.private.server/simple/your_package_name/
+Author: MindX
+Author-email: quyentm@mindx.com.vn
+License: LICENSE.txt
+License-File: LICENSE.txt
+Requires-Dist: pyspark ==3.5.1
+Requires-Dist: delta-spark ==3.1.0
+Requires-Dist: redis ==5.0.5
+Requires-Dist: esdbclient ==1.0.25
+Requires-Dist: kafka-python ==2.0.2
+
+## OK

mx_stream_core-0.1.2.dist-info/RECORD

@@ -0,0 +1,23 @@
+mx_stream_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mx_stream_core/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mx_stream_core/config/app.py,sha256=I5JT3NV_N2ISOv81tOWHyHtlZqLeOXcQxWuhU34MuXQ,105
+mx_stream_core/config/delta.py,sha256=yjGBnoslG8IW1eljEYaIA0pBujovRzZKEf5tgmqD0w0,260
+mx_stream_core/config/event_store.py,sha256=VySb1807-oLOpLLdrTulISUg5Ec0mD5H-Xp3sfWCYIk,119
+mx_stream_core/config/kafka.py,sha256=PpmGYovIMpesZQnZMbhJZt04dHNaznce58GwHOoXlUM,75
+mx_stream_core/config/layer_events.py,sha256=sk0l9eS3TbnmEGxgJKAom1Zw53JeDeEapkTTyZEZbZY,154
+mx_stream_core/config/s3.py,sha256=Cb4Qi3oQdw1RzHFCILAsdLPe35oBSQ-dpXbSjwe2JtE,267
+mx_stream_core/config/spark.py,sha256=gM2UnnWwsfKgjkBxaDERzY9y4dTnxRJzID4Ioh0tTrA,60
+mx_stream_core/config/streams.py,sha256=_t0jKGUjuT55Pc2_iC_nGRc_NxyMI4kC4DO37USvbeA,30
+mx_stream_core/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mx_stream_core/infrastructure/delta.py,sha256=Q_3c1zAvyfJwMz8jYIOU17NI477S8cyu7j5pEoH6nJA,226
+mx_stream_core/infrastructure/event_store.py,sha256=VYXWwTkhVhAFfUhGJPsZRyURaCdGiXdivHJDiyTsKhY,1193
+mx_stream_core/infrastructure/kafka.py,sha256=xPV1BQa62h9j2KHo7h86d7MuSh7vMZOdzwujhf4ae9g,4050
+mx_stream_core/infrastructure/redis.py,sha256=uHiUbwJdW09Hdbm3wZLmpEuB1XFnRm67G6weeTVpBeI,1508
+mx_stream_core/infrastructure/spark.py,sha256=XpBmllXpK2AEiDlng1qnZ-ZpYRAUIMgMJDJ6aH71w0g,1289
+mx_stream_core/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mx_stream_core/utils/cmd.py,sha256=KHdIjRpWlFO_m8p9zkawoBNOn8mIyZUL4U3fEsKsrCc,334
+mx_stream_core-0.1.2.dist-info/LICENSE.txt,sha256=VlM5vE0z1ygXtYMCQRLrf1zfPl7vAlLW7BucmpThK7M,2
+mx_stream_core-0.1.2.dist-info/METADATA,sha256=vCyNqeayroM14ypx1DbHvCD6K_vKQYDy9Sfu0NbQ7b8,453
+mx_stream_core-0.1.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+mx_stream_core-0.1.2.dist-info/top_level.txt,sha256=7P7-xBnxxYlylDW1z6XXlCWBZFn_rKCU8ESabosijcw,15
+mx_stream_core-0.1.2.dist-info/RECORD,,

mx_stream_core-0.1.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.43.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

mx_stream_core-0.1.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+mx_stream_core