pystrm 0.1.0__tar.gz

pystrm-0.1.0/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.3
+Name: pystrm
+Version: 0.1.0
+Summary: Python Stream module for api to kafka topic
+Requires-Dist: appdirs>=1.4.4
+Requires-Dist: attrs>=25.1.0
+Requires-Dist: confluent-kafka~=2.6.0
+Requires-Dist: fastavro>=1.10.0
+Requires-Dist: jsonschema>=4.23.0
+Requires-Dist: multiprocessing-logging>=0.3.4
+Requires-Dist: psycopg2-binary>=2.9.10
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: xxhash>=3.5.0
+Requires-Dist: yfinance>=0.2.65
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+Project for stream data from yfinance to Kafka stream.

pystrm-0.1.0/README.md

@@ -0,0 +1 @@
+Project for stream data from yfinance to Kafka stream.

pystrm-0.1.0/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "pystrm"
+version = "0.1.0"
+description = "Python Stream module for api to kafka topic"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "appdirs>=1.4.4",
+    "attrs>=25.1.0",
+    "confluent-kafka~=2.6.0",
+    "fastavro>=1.10.0",
+    "jsonschema>=4.23.0",
+    "multiprocessing-logging>=0.3.4",
+    "psycopg2-binary>=2.9.10",
+    "pyyaml>=6.0.2",
+    "xxhash>=3.5.0",
+    "yfinance>=0.2.65",
+]
+
+[project.scripts]
+pystrm = "pystrm.main:main"
+
+[tool.uv.build-backend]
+module-name = "pystrm"
+namespace = true
+
+[build-system]
+requires = ["uv_build>=0.9.17,<0.10.0"]
+build-backend = "uv_build"

pystrm-0.1.0/src/pystrm/kmain/kAdminstrator.py

@@ -0,0 +1,42 @@
+import logging
+from attrs import define
+from confluent_kafka.admin import AdminClient, NewTopic
+from pystrm.utils.constants import KAFKA_BROKERS
+
+logger = logging.getLogger(__name__)
+
+
+@define(kw_only=True)
+class KAdmin:
+    admin = AdminClient(KAFKA_BROKERS)
+
+    def topic_exists(self, topic: str) -> bool:
+        try:
+            all_topics = self.admin.list_topics()
+        except Exception as err:
+            logger.error(f"Failed to fetch list of topics. Error : {str(err)}")
+        return topic in all_topics.topics.keys()
+    
+    
+    def create_topic(self, topic: str, num_part: int = 1, replica: int = 1) -> None:
+        if not self.topic_exists(topic):
+            new_topic = [NewTopic(topic, num_partitions=num_part, replication_factor=replica)]
+            try:
+                self.admin.create_topics(new_topic)
+                logger.info(f"Topic {topic} has been created")
+            except Exception as err:
+                logger.error(f"Failed to create topic: {topic}. Error : {str(err)}")
+        else:
+            logger.warning(f"Topic {topic} already exists")
+
+    
+    def delete_topic(self, topics: list[str]) -> None:
+        for topic in topics:
+            if not self.topic_exists(topic):
+                try:
+                    self.admin.delete_topics([topic])
+                    logger.warning(f"'{topic}' topic deleted")
+                except Exception as err:
+                    logger.error(f"Failed to delete topic: {topic}. Error : {str(err)}")
+            else:
+                logger.info(f"Topic name {topic} does not exists")

pystrm-0.1.0/src/pystrm/kmain/kConsumer.py

@@ -0,0 +1,78 @@
+import logging
+from functools import lru_cache
+from attrs import define, field
+from json import dumps
+
+from confluent_kafka import Consumer
+from confluent_kafka.schema_registry.avro import AvroDeserializer
+from confluent_kafka.serialization import SerializationContext, MessageField
+from pystrm.kmain.kSchemaRegistry import KSR
+
+from src.pystrm.utils.constants import KAFKA_BROKERS
+from pystrm.utils.logger.logDecor import logtimer
+from pystrm.utils.confs import get_clientSchema
+
+
+logger = logging.getLogger(__name__)
+
+@lru_cache
+def schemaClient(topic: str, schema_type: str) -> KSR:
+
+    schema_str = get_clientSchema(topic, schema_type=schema_type)
+    schema_client = KSR(topic=topic, schema_str=dumps(schema_str), schema_type=schema_type)
+
+    return schema_client
+
+
+@define(kw_only=True)
+class kConsume:
+    topic: str = field(eq=str)
+    groupId: str = field(eq=str, default=None)
+    schema_type: str = field(eq=str, default=None)
+
+    @logtimer
+    def consume_message(self):
+
+        consumer = Consumer({"bootstrap.servers": KAFKA_BROKERS, "group.id": self.groupId})
+        consumer.subscribe([self.topic])
+
+        try:
+            while True:
+                msg = consumer.poll(0.5)
+                if msg is None:
+                    continue
+                if msg.error():
+                    logger.error("Failed to recieve message. Error: %s" % (str(msg.error())))
+                    continue
+                logger.info("Message Consumed: %s" % (str(msg.value())))
+        except KeyboardInterrupt:
+            logger.warning("Keyboard Interrupt happened")
+        finally:
+            consumer.close()
+
+
+    @logtimer
+    def consume_serialized_message(self):
+
+        consumer = Consumer({"bootstrap.servers": KAFKA_BROKERS, "group.id": self.groupId})
+        consumer.subscribe([self.topic])
+        schema_client = schemaClient(self.topic, self.schema_type)
+        schema_str = schema_client.get_schema_str()
+        value_deserializer = AvroDeserializer(schema_client, schema_str)
+
+        try:
+            while True:
+                msg = consumer.poll(0.5)
+                if msg is None:
+                    continue
+                if msg.error():
+                    logger.error("Failed to recieve message. Error: %s" % (str(msg.error())))
+                    continue
+                # logger.info("Message Consumed: %s" % (str(msg.value())))
+                message = msg.value()
+                deserialized_message = value_deserializer(message, SerializationContext(self.topic, MessageField.VALUE))
+                logger.info(f"Message Consumed: {deserialized_message}")
+        except KeyboardInterrupt:
+            logger.warning("Keyboard Interrupt happened")
+        finally:
+            consumer.close()

pystrm-0.1.0/src/pystrm/kmain/kProducer.py

@@ -0,0 +1,98 @@
+import logging
+from functools import lru_cache
+from attrs import define, field
+from json import dumps
+from uuid import uuid4
+from jsonschema import validate as jsonValidate, ValidationError as jsonValidationError
+from fastavro import validate as avroValidate
+from fastavro.validation import ValidationError as avroValidationError
+
+from confluent_kafka import Producer
+from confluent_kafka.serialization import (
+    IntegerSerializer,
+    StringSerializer,
+    SerializationContext,
+    MessageField,
+)
+from confluent_kafka.schema_registry import SchemaRegistryClient
+from confluent_kafka.schema_registry.avro import AvroSerializer
+from confluent_kafka.schema_registry.json_schema import JSONSerializer
+from numpy import int64
+
+from pystrm.utils.constants import KAFKA_BROKERS, KAFKA_SCHEMA_CLIENT
+from pystrm.utils.logger.logDecor import logtimer
+
+logger = logging.getLogger(__name__)
+
+@lru_cache
+def get_producer():
+    return Producer(KAFKA_BROKERS)
+
+
+@lru_cache
+def get_ksr():
+    return SchemaRegistryClient(KAFKA_SCHEMA_CLIENT)
+
+
+@define(kw_only=True)
+class Kprod:
+    topic: str = field(eq=str)
+
+
+    @logtimer
+    def acked(self, err, msg) -> None:
+        if err is not None:
+            logger.error("Failed to deliv" \
+            "er message: %s: %s" % (str(msg), str(err)))
+        else:
+            logger.info("Message produced: %s" % (str(msg)))
+
+        return None
+
+
+    @logtimer
+    def prodDataWithJsonSerial(self, data: str, mykey: int | str | None) -> None:
+        
+        producer = get_producer()
+
+        try:
+            producer.produce(self.topic, key=mykey, value=dumps(data).encode(), on_delivery=self.acked)
+
+            producer.flush()
+        except Exception as err:
+            logger.error(f"Error sending message: {str(err)}")
+            return None
+
+
+    @logtimer
+    def prodDataWithSerialSchema(self, schema, data: dict[str, int64 | str], mykey: int | str, schema_type: str = "JSON") -> None:
+
+        producer = get_producer()
+        schema_client = get_ksr()
+
+        try:
+            jsonValidate(data, schema) if schema_type == "JSON" else avroValidate(data, schema)
+            logger.info("Schema Validated")
+
+            key_serializer = IntegerSerializer() if isinstance(mykey, int) else StringSerializer()
+            value_serializer = JSONSerializer(dumps(schema), schema_client) if schema_type == "JSON" else AvroSerializer(schema_client, dumps(schema))
+
+            producer.produce(
+                topic=self.topic,
+                key=key_serializer(mykey),
+                value=value_serializer(data, SerializationContext(self.topic, MessageField.VALUE)),
+                headers={"correlation_id": key_serializer(str(uuid4()))},
+                on_delivery=self.acked
+            )
+            
+            producer.flush()
+        except jsonValidationError as err:
+            logger.error(f"Json Validation Error: {str(err)}")
+            return None
+        except avroValidationError as err:
+            logger.error(f"Avro Validation Error: {str(err)}")
+            return None
+        except Exception as err:
+            logger.error(f"Error sending message: {str(err)}")
+            return None
+

pystrm-0.1.0/src/pystrm/kmain/kSchemaRegistry.py

@@ -0,0 +1,46 @@
+import logging
+from attrs import define, field
+from confluent_kafka.schema_registry import SchemaRegistryClient, Schema
+from confluent_kafka.schema_registry.error import SchemaRegistryError
+from pystrm.utils.constants import KAFKA_SCHEMA_CLIENT
+
+logger = logging.getLogger(__name__)
+
+
+@define(kw_only=True)
+class KSR:
+    topic: str = field(eq=str)
+    schema_str: str = field(eq=str)
+    schema_type: str = field(eq=str, default="AVRO")
+    schema_client = SchemaRegistryClient(KAFKA_SCHEMA_CLIENT)
+    
+
+    def get_schema_version(self):
+        try:
+            schema_version = self.schema_client.get_latest_version(self.topic)
+            logger.info(f"{self.topic} schema does exists.")
+            return schema_version.schema_id
+        except SchemaRegistryError as err:
+            logger.warning("Schema does not exists : " + str(err))
+            return False
+    
+    
+    def get_schema_str(self):
+        try:
+            schema_id = self.get_schema_version()
+            schema = self.schema_client.get_schema(schema_id)
+            return schema.schema_str
+        except SchemaRegistryError as err:
+            logger.warning(f"Some error occured while fetching schema_id '{schema_id}': " + str(err))
+
+
+    def register_schema(self):
+        if not self.get_schema_version():
+            try:
+                schema = Schema(self.schema_str, self.schema_type)
+                self.schema_client.register_schema(self.topic, schema)
+                logger.info("Schema Registered")
+            except SchemaRegistryError as err:
+                logger.warning("Schema registry failed. Error: " + str(err))
+        else:
+            logger.warning(f"{self.topic} already registered")

pystrm-0.1.0/src/pystrm/main.py

@@ -0,0 +1,68 @@
+import logging
+import logging.config
+import sys
+from pprint import pformat
+from time import sleep
+from typing import Any
+from datetime import datetime
+from pathlib import Path
+from pystrm.utils.mainCalls.yfUtils import  getLiveTickData
+from pystrm.utils.confs import fetch_conf, fetch_prop
+from pystrm.utils.constants import RUN_ID
+from pystrm.utils.logger.logDecor import inOutLog
+from multiprocessing_logging import install_mp_handler
+
+
+@inOutLog
+def main() -> None: 
+
+    Path(Path.cwd()/'logs').mkdir(exist_ok=True)
+
+    config: dict[str, Any] = fetch_conf()['Logging']
+    config['handlers']['file_json']['filename'] += datetime.now().strftime('%Y-%m-%d.json')
+    
+    logging.config.dictConfig(config)
+    logger = logging.getLogger()
+
+    install_mp_handler(logger)
+    
+    logger.info("Intitiating program with run_id : " + str(RUN_ID))
+
+    __method_to_excute = {
+        "liveYfinanaceTick": getLiveTickData
+    }
+    
+    try:
+        if len(sys.argv) != 3:
+            raise TypeError
+
+        mthd = sys.argv[1].strip()
+        conf_key = sys.argv[2].strip()
+
+        if mthd not in __method_to_excute.keys():
+            msg = "List of operation mentioned in dictionary for this package"
+            raise KeyError
+        else:
+            logger.info(f"Operation {mthd} exists. Validating other input")
+        
+        if (conf_key.split('.')[0] not in fetch_prop().keys()) or (conf_key.split('.')[1] not in fetch_prop()[conf_key.split('.')[0]]):
+            msg = f"key:{conf_key.split('.')[0]} and value: {conf_key.split('.')[1]} pair does not exists in tables.yml"
+            raise KeyError
+        else:
+            logger.info(f"Configuration found in tables.yml for {conf_key}")
+            logger.info("Config found for this operation from tables.yml are as below:")
+            logger.info(f"\n{pformat(fetch_prop()[conf_key.split('.')[0]], indent=4)}")
+
+        return __method_to_excute[mthd](conf_key)
+    except TypeError:
+        logger.critical(f"main() function takes exactly 2 arguments ({len(sys.argv[1:])} given)")
+        sleep(1)
+        sys.exit(1)
+    except KeyError:
+        logger.critical(f'Key not found: {msg}')
+        sleep(1)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

pystrm-0.1.0/src/pystrm/stream/yfStream/ticksStream.py

@@ -0,0 +1,249 @@
+import logging
+import asyncio
+from operator import itemgetter
+from multiprocessing import Pool
+from time import sleep
+from typing import Any
+import os
+
+from numpy import int64
+
+from pystrm.kmain.kProducer import Kprod
+# from pystrm.schema.yfSchema.ticksSchemaObj import fastInfoSchemaObject
+
+from pystrm.utils.confs import fetch_conf, get_clientSchema, streamConf
+from pystrm.utils.kUtils import createKtopic, registerClientSchema
+from pystrm.utils.logger.logDecor import logtimer, inOutLog
+
+
+import appdirs as ad
+ad.user_cache_dir = lambda *args: "/tmp"
+import yfinance as yf  # noqa: E402
+
+logger = logging.getLogger(__name__)
+
+
+def dataDupCheck(data_dct: dict[str, Any], typ: str, data: dict[str, int64 | str], symb: str) -> bool:
+    
+    if (typ + "." + symb) in data_dct.keys():
+        if data_dct[typ + "." + symb] == data:
+            return False
+        else:
+            data_dct[typ + "." + symb] = data
+            return True
+    else:
+        data_dct[typ + "." + symb] = data
+        return True
+
+
+async def validSend(kobj: Kprod, typ: str, data_dct: dict[str, Any], symb: str, data: dict[str, int64 | str], schema_type: str, schema: object) -> None:
+    if dataDupCheck(data_dct=data_dct, typ=typ, data=data, symb=symb):
+        logger.info(f"New record found for symbol {symb}")
+        kobj.prodDataWithSerialSchema(schema=schema, data=data, mykey=symb, schema_type=schema_type)
+    else:
+        logger.info(f"Duplicate record found for symbol {symb}")
+
+
+@logtimer
+async def asyncTicker(kobj: Kprod, symb: str, meth: str):
+    data = yf.Ticker(symb+".NS").__getattribute__(meth)
+    kobj.prodDataWithJsonSerial(data=data, mykey=symb)
+
+
+@inOutLog
+async def ticker(kobj: Kprod, symbol: list[str], param_dct: dict[str, Any]) -> None:
+    """_summary_
+
+    Args:
+        symbol (str): symbol for which ticker data will be generated
+    """
+    
+    indx: int = 0
+
+    if len(symbol) == 0:
+        logger.info("Symbol list is of zero size")
+        return
+
+    while True: 
+        indx = indx % len(symbol)
+
+        if indx == 0:
+            sleep(1)
+
+        try:
+            await asyncTicker(kobj, symbol[indx], param_dct['prop_key'])
+        except KeyboardInterrupt:
+            logger.error("KeyboardInterrrupt happened")
+            break
+        except Exception as e:
+            logger.error(str(e))
+        
+        if param_dct['type'] != 'Streaming':
+            sleep(5)
+            break
+        
+        indx += 1
+    
+    return None
+
+
+@logtimer
+async def asyncFastInfo(symb: str, meth: list[str]) -> object:
+    data = dict(zip([item.lower() for item in meth], list(itemgetter(*meth)(yf.Ticker(symb+".NS").fast_info))))
+    # dataObjt = fastInfoSchemaObject(*data)
+    return data
+
+
+@inOutLog
+async def fastInfo(kobj: Kprod, symbol: list[str], param_dct: dict[str, Any]) -> None:
+    """_summary_
+
+    Args:
+        symbol (str): symbol for which ticker data will be generated
+    """
+
+    indx: int = 0
+
+    if len(symbol) == 0:
+        logger.info("Symbol list is of zero size")
+        return
+
+    schema = get_clientSchema(param_dct['prop_key'], param_dct['schema_type'])
+
+    dupCheck = dict()
+
+    while True: 
+        print(dupCheck)
+        indx = indx % len(symbol)
+
+        if indx == 0:
+            sleep(1)
+
+        try:  
+            data = await asyncFastInfo(symbol[indx], param_dct['infolst'])
+            await validSend(kobj, "fastInfo", dupCheck, symbol[indx], data, param_dct['schema_type'], schema)
+        except KeyboardInterrupt:
+            logger.warning("KeyboardInterrrupt happened")
+        except Exception as e:
+            logger.error(str(e))
+        
+        if param_dct['type'] != 'Streaming':
+            sleep(5)
+            break
+        
+        indx += 1
+
+    return None
+
+
+# @inOutLog
+# @logtimer
+# async def multiTicker(kobj: Kprod, symbol: list[str], param_dct: dict[str, Any]) -> None:
+#     """_summary_
+
+#     Args:
+#         symbol (str): symbol for which ticker data will be generated
+
+#     Returns:
+#         dict[str, Any]: Fetch ticker data 
+#     """
+
+#     # schema_client = SchemaRegistryClient(KAFKA_SCHEMA_CLIENT)
+
+#     indx: int = 0
+
+#     if len(symbol) == 0:
+#         logger.info("Symbol list is of zero size")
+#         return
+
+#     while True: 
+#         indx = indx % len(symbol)
+#         try:
+#             data = await asyncFastInfo(symbol[indx],  param_dct['infolst'])
+#             await kobj.prodDataWithAvroSerialSchema(value=data, key=symbol[indx])
+#             sleep(1)
+#         except KeyboardInterrupt:
+#             logger.warning("KeyboardInterrrupt happened")
+#             break
+#         except Exception as e:
+#             logger.error(str(e))
+        
+#         if param_dct['type'] != 'Streaming':
+#             sleep(5)
+#             break
+        
+#         indx += 1
+#     return None
+
+
+@inOutLog
+@logtimer
+def procHandler(param_dct: dict[str, Any], symb: list[str]) -> None:
+    """Fetch data from Yahoo Finance till market close in multiprocess and concurrrent waiting manner
+
+    Args:
+        param_dct (dict[str, str]): Parameter dictionary for execution
+        symb (list[str]): List of stock symbols for fetch data
+    """
+    
+
+    kobj = Kprod(topic=param_dct['prop_key'])
+
+    __EXECUTE_METHOD = {
+        "tick": ticker
+        # ,"multitick": multiTicker
+        ,"fastinfo": fastInfo
+    }
+
+    asyncio.run(__EXECUTE_METHOD[param_dct['ldt']](kobj, symb, param_dct))
+    return None
+
+
+def process_status(err):
+    if err is not None:
+        logger.error("Failed process: %s" % (str(err)))
+    else:
+        logger.info("Process Success")
+        
+
+@inOutLog
+def getStreamData(key: str) -> None:
+    """Generate data from Jugaad package
+
+    Args:
+        key (str): Take config key as input
+    """
+
+    num_proc, num_process, symbols, prop_dict = streamConf(key)
+
+    createKtopic(topic=prop_dict['prop_key'], num_part=9, replica=3)
+
+    if 'schema_type' in prop_dict.keys():
+        registerClientSchema(topic=prop_dict['prop_key'], schema_type=prop_dict['schema_type'])
+
+    try:
+        pool = Pool(processes=os.cpu_count())
+
+        input_list = list()
+        for i in range(1, num_proc+1):
+            if i == num_proc:
+                input_list.append((prop_dict, symbols[(i - 1)*num_process:]))
+            else:
+                input_list.append((prop_dict, symbols[(i-1)*num_process:i*num_process]))
+            
+        ar = pool.starmap_async(procHandler, input_list, error_callback=process_status)
+        ar.wait(timeout=fetch_conf()['Market']['LiveTime'])
+        if ar.ready():
+            logger.info("All task finished")
+        else:
+            logger.error("Some task still running")
+            pool.terminate()
+            return None
+    except KeyboardInterrupt as e:
+        logger.warning("Keyboard Interrupt : " + str(e))
+        pool.terminate()
+        return None
+    except Exception as e:
+        logger.error(str(e))
+        pool.terminate()
+        return None

pystrm-0.1.0/src/pystrm/utils/confs.py

@@ -0,0 +1,94 @@
+from typing import Any
+from yaml import safe_load
+from fastavro.schema import load_schema
+import json
+#import requests
+#from utils import BASE_URL, HEADERS, NIFTY50
+
+
+def fetch_prop() -> dict[str, Any]:
+
+    with open('../config/tables.yaml', 'r') as tblinfo:
+        prop_dict = safe_load(tblinfo.read())
+
+    return prop_dict
+
+
+def fetch_conf() -> dict[str, Any]:
+
+    with open('../config/config.yaml', 'r') as cfginfo:
+        conf_dict = safe_load(cfginfo.read())
+
+    return conf_dict
+
+
+def param_init(key: str) -> Any:
+
+    conf_key = key.split('.')[0]
+    prop_key = key.split('.')[1]
+    conf_dict = fetch_conf()[conf_key]
+    prop_dict = fetch_prop()[conf_key][prop_key]
+
+    return conf_key, prop_key, conf_dict, prop_dict
+
+
+def fetch_db_dtl(db_type: str) -> dict[str, str | int]:
+    db_dtl = fetch_conf()[db_type]
+    return db_dtl
+
+
+def get_clientSchema(fl_nm: str, schema_type: str):
+
+    path = fetch_conf()["Kafka"]["schema"][schema_type.lower()]
+
+    if schema_type == "JSON":
+        fl_nm += ".json"
+        with open(path + fl_nm) as fl:
+            schema = json.load(fl)
+    else:
+        fl_nm += ".avsc"
+        schema = load_schema(path + fl_nm)
+
+    return schema
+
+def streamConf(key: str) -> tuple[int, int, list[str], dict[str, Any]]:
+
+    conf_key, prop_key, conf_dict, prop_dict = param_init(key)
+    symbols: list[str] = fetch_conf()['Market']['Symbols'] 
+    
+    prop_dict['prop_key'] = prop_key.lower()
+
+    num_proc: int = int(conf_dict['num_process'])
+
+    num_process: int = len(symbols)//num_proc
+
+    if len(symbols) % num_proc == 0:
+        num_proc = num_proc
+    else:
+        num_proc = num_proc + 1
+    
+    return (num_proc, num_process, symbols, prop_dict)
+
+'''@logtimer
+@staticmethod
+def niftySymbols(url: str) -> list[str]:
+    
+    session = requests.Session()
+    r = session.get(BASE_URL, headers=HEADERS, timeout=5)
+    cookies = dict(r.cookies)
+    
+    response = session.get(url, timeout=5, headers=HEADERS, cookies=cookies)
+    content = response.content.decode('utf-8')
+
+    columns=['Company Name', 'Industry', 'Symbol', 'Series', 'ISIN Code']
+    data_lst = [x.strip().split(',') for x in content.splitlines() if x.strip().split(',') != columns]              
+
+    df=pd.DataFrame(data_lst, columns=columns)
+    symbols_lst = df['Symbol'].tolist()
+    
+    return symbols_lst'''
+    
+
+#print(niftySymbols(NIFTY50))
+#print(param_init('JugaadData.PriceInfo'))
+#print(fetch_conf())

pystrm-0.1.0/src/pystrm/utils/constants.py

@@ -0,0 +1,17 @@
+from typing import Final
+from datetime import datetime
+from pystrm.utils.confs import fetch_conf
+from socket import gethostname
+from xxhash import xxh32_intdigest
+
+
+RUN_ID: Final[int] = int(datetime.now().strftime('%Y%m%d')  + str(xxh32_intdigest("pystream" + datetime.now().strftime('%H:%M:%S.%f'))))
+
+CURRENT_DATE: Final[str] = datetime.today().strftime('%Y-%m-%d')
+
+KAFKA_BROKERS: Final[dict[str, str]] = fetch_conf()['Kafka']['kafka-broker-conf'] | {'client.id': gethostname()}
+
+KAFKA_SCHEMA_CLIENT: Final[dict[str, str]] = fetch_conf()['Kafka']['kafka-schema-client-conf']
+
+
+__all__ = ('CURRENT_DATE', 'KAFKA_BROKERS', 'KAFKA_SCHEMA_CLIENT')

pystrm-0.1.0/src/pystrm/utils/kUtils.py

@@ -0,0 +1,24 @@
+from json import dumps
+from pystrm.kmain.kAdminstrator import KAdmin
+from pystrm.kmain.kSchemaRegistry import KSR
+from pystrm.utils.confs import get_clientSchema
+
+
+def createKtopic(topic: str, num_part: int = 1, replica: int = 1) -> None:
+
+    adm = KAdmin()
+    adm.create_topic(topic=topic, num_part=num_part, replica=replica)   
+
+    return None
+
+
+def registerClientSchema(topic: str, schema_type: str) -> None:
+
+    schema_str = get_clientSchema(topic, schema_type=schema_type)
+    schema_register = KSR(topic=topic, schema_str=dumps(schema_str), schema_type=schema_type)
+    schema_register.register_schema()
+
+    return None
+
+def to_dict(obj: object, ctx):
+    return dict(vars(obj))

pystrm-0.1.0/src/pystrm/utils/logger/QueueListenerHandler.py

@@ -0,0 +1,38 @@
+from logging.config import ConvertingList
+from logging.handlers import QueueHandler, QueueListener
+from queue import Queue
+from atexit import register
+
+
+def _resolve_handlers(lst: ConvertingList):
+    if not isinstance(lst, ConvertingList):
+        return lst
+
+    # Indexing the list performs the evaluation.
+    return [lst[i] for i in range(len(lst))]
+
+
+class QueueListenerHandler(QueueHandler):
+
+    def __init__(self, handlers, respect_handler_level=True, auto_run=True, queue=Queue(-1)):
+        super().__init__(queue)
+        handlers = _resolve_handlers(handlers)
+        self._listener = QueueListener(
+            self.queue,
+            *handlers,
+            respect_handler_level=respect_handler_level)
+        if auto_run:
+            self.start()
+            register(self.stop)
+
+
+    def start(self):
+        self._listener.start()
+
+
+    def stop(self):
+        self._listener.stop()
+
+
+    def emit(self, record):
+        return super().emit(record)

pystrm-0.1.0/src/pystrm/utils/logger/jsonLogging.py

@@ -0,0 +1,80 @@
+import datetime as dt
+import json
+import logging
+from pystrm.utils.constants import RUN_ID
+
+LOG_RECORD_BUILTIN_ATTRS = {
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "msecs",
+    "message",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+    "taskName",
+}
+
+
+class logJSONFormatter(logging.Formatter):
+    def __init__(
+        self,
+        *,
+        fmt_keys: dict[str, str] | None = None,
+    ):
+        super().__init__()
+        self.fmt_keys = fmt_keys if fmt_keys is not None else {}
+
+
+    def format(self, record: logging.LogRecord) -> str:
+        message = self._prepare_log_dict(record)
+        return json.dumps(message, default=str)
+
+    def _prepare_log_dict(self, record: logging.LogRecord):
+        always_fields = {
+            "run_id": RUN_ID,
+            "jobtype": "pyStream",
+            "message": record.getMessage(),
+            "timestamp": dt.datetime.fromtimestamp(
+                record.created, tz=None
+            ).isoformat(),
+        }
+        if record.exc_info is not None:
+            always_fields["exc_info"] = self.formatException(record.exc_info)
+
+        if record.stack_info is not None:
+            always_fields["stack_info"] = self.formatStack(record.stack_info)
+
+        message = {
+            key: msg_val
+            if (msg_val := always_fields.pop(val, None)) is not None
+            else getattr(record, val)
+            for key, val in self.fmt_keys.items()
+        }
+        message.update(always_fields)
+
+        for key, val in record.__dict__.items():
+            if key not in LOG_RECORD_BUILTIN_ATTRS:
+                message[key] = val
+        
+        return message
+
+
+class NonErrorFilter(logging.Filter):
+
+    def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord:
+        return record.levelno <= logging.INFO

pystrm-0.1.0/src/pystrm/utils/logger/logDecor.py

@@ -0,0 +1,33 @@
+import logging
+import inspect
+from time import perf_counter
+from typing import Any, Callable
+from functools import wraps, partial
+
+logger = logging.getLogger(__name__)
+
+
+def logtimer(func: Callable[..., Any]) -> Callable[..., Any]:
+    @wraps(func)
+    def wrapper(*args: Any, **kwargs: Any):
+        start = perf_counter()
+        result = func(*args, **kwargs)
+        logger.info(f"Execution of {func.__name__} took: {(perf_counter() - start):.6f} sec")
+        return result
+    return wrapper
+
+
+def funcLifeLog(func: Callable[..., Any], logger: logging.Logger) -> Callable[..., Any]:
+    @wraps(func)
+    def wrapper(*args: Any, **kwargs: Any):
+        logger.info(f"Calling for execution of {func.__name__}")
+        func_args = inspect.signature(func).bind(*args, **kwargs).arguments
+        func_args_str = ", ".join(map("{0[0]} = {0[1]!r}".format, func_args.items()))
+        print(f"{func.__module__}.{func.__qualname__} ( {func_args_str} )")
+        value = func(*args, **kwargs)
+        logger.info(f"Finished execution of {func.__name__}")
+        return value
+    return wrapper
+
+
+inOutLog = partial(funcLifeLog, logger=logger)

pystrm-0.1.0/src/pystrm/utils/logger/psgLogging.py

@@ -0,0 +1,61 @@
+import logging
+import psycopg2 as psg
+from pystrm.utils.constants import RUN_ID
+import datetime as dt
+from pystrm.utils.confs import fetch_db_dtl
+
+
+postgres_dtl = fetch_db_dtl('Postgresql')
+
+try:
+    connection = psg.connect(host=postgres_dtl['hostname'], dbname=postgres_dtl['db'], user=postgres_dtl['username'], password=postgres_dtl['password'], port=postgres_dtl['port'])
+except (Exception, psg.DatabaseError) as e:
+    print("RDBMS connection failed to database : " + str(e))
+else:
+    cursor = connection.cursor()
+
+
+def execute_query(qry: str) -> None:
+    try:
+        cursor.execute(qry)
+        connection.commit()
+    except Exception as e:
+        print(f'Error {e}')
+        print('Anything else that you feel is useful')
+        connection.rollback()
+
+
+def close_connection():
+    cursor.close()
+    connection.close()
+
+
+log_tbl: str = f"""CREATE TABLE IF NOT EXISTS {postgres_dtl['schema']}.{postgres_dtl['table']} (
+    run_id BIGINT NOT NULL,
+    job_type VARCHAR(200) NOT NULL,
+    level_name VARCHAR(200) NOT NULL,
+    message TEXT NOT NULL,
+    logger VARCHAR(200) NOT NULL,
+    module VARCHAR(200) NOT NULL,
+    function_name VARCHAR(200) NOT NULL,
+    filename VARCHAR(200) NOT NULL,
+    line_num VARCHAR(200) NOT NULL,
+    log_time VARCHAR(200) NOT NULL,
+    PRIMARY KEY(run_id, log_time)
+);"""
+
+execute_query(log_tbl)
+
+
+class CustomHandler(logging.StreamHandler):
+
+    def __init__(self):
+        super().__init__()
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if record:
+            msg = record.getMessage()
+            if msg.startswith('"') and msg.endswith('"'):
+                msg = msg[1:-1]
+            msg = msg.replace("'", "`")
+            execute_query(f"INSERT INTO {postgres_dtl['schema']}.{postgres_dtl['table']} (run_id, job_type, level_name, message, logger, module, function_name, filename, line_num, log_time) VALUES ({RUN_ID},'pyStream','{record.levelname}','{msg}','{record.name}','{record.module}','{record.funcName}','{record.filename}',{record.lineno},'{dt.datetime.fromtimestamp(record.created, tz=None).isoformat()}')")

pystrm-0.1.0/src/pystrm/utils/mainCalls/yfUtils.py

@@ -0,0 +1,13 @@
+from pystrm.stream.yfStream.ticksStream import getStreamData
+from pystrm.utils.logger.logDecor import inOutLog
+
+@inOutLog
+@staticmethod
+def getLiveTickData(key: str) -> None:
+    """_summary_
+
+    Args:
+        key (str): Fetch live quote data from package yfinance and push it to kafka producer. 
+    """
+
+    getStreamData(key=key)