pynop 0.1.0__tar.gz

pynop-0.1.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: pynop
+Version: 0.1.0
+Summary: Shared DB/core + NopCommerce tooling for ABC StoreDataSync
+Author-email: ABC Sync <noreply@example.com>
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# pynop (ABC StoreDataSync)
+
+This package contains the shared `core` utilities (DB access, Biz/log framework) and the `NopTool`/`NopQuery` helpers used by the `sync` project.
+

pynop-0.1.0/README.md

@@ -0,0 +1,4 @@
+# pynop (ABC StoreDataSync)
+
+This package contains the shared `core` utilities (DB access, Biz/log framework) and the `NopTool`/`NopQuery` helpers used by the `sync` project.
+

pynop-0.1.0/pynop/__init__.py

@@ -0,0 +1,4 @@
+"""
+Shared DB/core layer + NopCommerce DB tooling for StoreDataSync.
+"""
+

pynop-0.1.0/pynop/core/__init__.py

@@ -0,0 +1,4 @@
+"""
+Shared core layer used by the StoreDataSync project.
+"""
+

pynop-0.1.0/pynop/core/biz.py

@@ -0,0 +1,95 @@
+import functools
+import traceback
+from datetime import datetime
+
+from .log import DataLogger
+
+import settings
+from .exceptions import BizError
+
+
+def timestamp():
+    """Return current timestamp as a formatted string."""
+    return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+
+def mark(func):
+    """Decorator to print markers with timestamps before, after, and on exceptions."""
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        # assume first arg is self
+        self = args[0]
+        logger = getattr(self, "logger", None)
+
+        start_msg = f"[{timestamp()}] --- START {func.__qualname__} ---"
+        end_msg = f"[{timestamp()}] --- END {func.__qualname__} ---"
+
+        try:
+            if logger:
+                logger.info(start_msg)
+            else:
+                print(start_msg)
+
+            result = func(*args, **kwargs)
+
+            if logger:
+                logger.info(end_msg)
+            else:
+                print(end_msg)
+
+            return result
+        except Exception as e:
+            err_msg = f"[{timestamp()}] !!! ERROR in {func.__qualname__}: {e}"
+            if logger:
+                logger.error(err_msg)
+            else:
+                print(err_msg)
+            traceback.print_exc()
+            raise
+
+        return wrapper
+
+
+def auto_mark_methods(cls):
+    """Class decorator to apply `mark` to all methods of a class."""
+    for attr_name, attr_value in cls.__dict__.items():
+        if callable(attr_value) and not attr_name.startswith("__"):
+            setattr(cls, attr_name, mark(attr_value))
+    return cls
+
+
+class Biz:
+    """
+    Abstract class to describe business logic of sync program.
+    """
+
+    def __init__(self, settings):
+        log_level = settings.LOG_LEVEL_OPTION
+        log_dir_path = settings.LOG_DIR_PATH
+        app_started_time = settings.APP_STARTED_TIME
+        self.logger = DataLogger(str(type(self).__name__), log_level, log_dir_path, app_started_time)
+
+        self.reports = {}
+
+    def start_biz(self):
+        pass
+
+    def end_biz(self):
+        self.logger.log_divider()
+        self.logger.log_biz_msg("Biz Reports", str(self.reports))
+        self.logger.log_current_time()
+        self.logger.log_divider()
+        self.logger.cleanup_expired_log(settings.LOG_DIR_PATH, settings.LOG_MAX_RETENTION_DAYS)
+        self.logger.end()
+
+    def run(self):
+        try:
+            self.start_biz()
+            self.end_biz()
+        except Exception as e:
+            self.logger.log_biz_msg("Biz Error", str(e))
+            raise e
+        finally:
+            pass
+

pynop-0.1.0/pynop/core/config.py

@@ -0,0 +1,6 @@
+PYODBC_MSSQL_CONNECTION_STRING = "DRIVER={ODBC Driver 17 for SQL Server};SERVER=%s;DATABASE=%s;UID=%s;PWD=%s"
+#PYODBC_MSSQL_CONNECTION_STRING = "DRIVER={SQL Server};SERVER=%s;DATABASE=%s;UID=%s;PWD=%s"
+PYODBC_MYSQL_CONNECTION_STRING = "DRIVER={MySQL ODBC 8.0 Unicode Driver};SERVER=%s;DATABASE=%s;UID=%s;PWD=%s"
+PYMONGO_CONNECTION_STRING = "mongodb://{server}:27017/"
+#PYMONGO_CONNECTION_STRING = "mongodb+srv://{username}:{password}@{server}/{db_name}"
+

pynop-0.1.0/pynop/core/db.py

@@ -0,0 +1,160 @@
+import pyodbc
+from pymongo import MongoClient
+from .config import *
+from settings import DB_CONFIG
+
+
+class DB:
+    _conn = None
+    _cursor = None
+
+    def __init__(self, db_engine, db_server, db_name, db_username, db_password):
+        if db_engine == "MONGO":
+            self._conn = MongoClient(PYMONGO_CONNECTION_STRING.format(server=db_server))
+            self._cursor = self._conn[db_name]
+        else:
+            conn_str = ""
+
+            if db_engine == "MSSQL":
+                conn_str = PYODBC_MSSQL_CONNECTION_STRING % (
+                    db_server,
+                    db_name,
+                    db_username,
+                    db_password,
+                )
+            elif db_engine == "MYSQL":
+                conn_str = PYODBC_MYSQL_CONNECTION_STRING % (
+                    db_server,
+                    db_name,
+                    db_username,
+                    db_password,
+                )
+            elif db_engine == "DSN":
+                conn_str = db_server
+
+            self._conn = pyodbc.connect(conn_str)
+            self._cursor = self._conn.cursor()
+
+        return
+
+
+class AutoDao:
+    def __init__(self, db_type="", db_engine="", db_server="", db_name="", db_username="", db_password=""):
+        env = DB_CONFIG["env"]
+
+        if db_type and db_type in DB_CONFIG[env].keys():
+            self.db = DB(
+                DB_CONFIG[env][db_type]["db_engine"],
+                DB_CONFIG[env][db_type]["db_server"],
+                DB_CONFIG[env][db_type]["db_name"],
+                DB_CONFIG[env][db_type]["db_username"],
+                DB_CONFIG[env][db_type]["db_password"],
+            )
+        else:
+            self.db = DB(db_type, db_server, db_name, db_username, db_password)
+
+    def get_hashmap(self, sql, param=[]):
+        result = {}
+        self.db._cursor.execute(sql, param)
+        for a in self.db._cursor.fetchall():
+            result[a[0]] = a[1]
+        return result
+
+    def get_hashtable(self, sql, param=[], key=0):
+        result = {}
+        self.db._cursor.execute(sql, param)
+        result_rows = self.db._cursor.fetchall()
+        if result_rows:
+            result_cols = [column[0] for column in self.db._cursor.description]
+            for row in result_rows:
+                result[row[key]] = dict(zip(result_cols, row))
+        return result
+
+    def get_hasharrtable(self, sql, param=[], key=0):
+        result = {}
+        self.db._cursor.execute(sql, param)
+        result_rows = self.db._cursor.fetchall()
+        if result_rows:
+            result_cols = [column[0] for column in self.db._cursor.description]
+            for row in result_rows:
+                row_key = row[key]
+                if row_key not in result:
+                    result[row_key] = []
+                result[row_key].append(dict(zip(result_cols, row)))
+        return result
+
+    def get_rows_and_keys(self, sql, param=[], count=0):
+        self.db._cursor.execute(sql, param)
+        if count:
+            result = self.db._cursor.fetchmany(count)
+        else:
+            result = self.db._cursor.fetchall()
+
+        result_cols = [column[0] for column in self.db._cursor.description]
+        return result, result_cols
+
+    def get_one_row_dict(self, sql, param=[]):
+        self.db._cursor.execute(sql, param)
+        result = self.db._cursor.fetchone()
+        if result:
+            result_cols = [column[0] for column in self.db._cursor.description]
+            result = dict(zip(result_cols, result))
+        return result
+
+    def get_one_col_list(self, sql, param=[], count=0):
+        self.db._cursor.execute(sql, param)
+        result = []
+        if count:
+            result = self.db._cursor.fetchmany(count)
+        else:
+            result = self.db._cursor.fetchall()
+        return [row[0] for row in result]
+
+    def get_one_value(self, sql, param=[]):
+        self.db._cursor.execute(sql, param)
+        result = self.db._cursor.fetchone()
+        if result:
+            result = result[0]
+        return result
+
+    def get_list(self, sql, param=[]):
+        result = []
+        self.db._cursor.execute(sql, param)
+        result_rows = self.db._cursor.fetchall()
+        if result_rows:
+            result_cols = [column[0] for column in self.db._cursor.description]
+            for row in result_rows:
+                result.append(dict(zip(result_cols, row)))
+        return result
+
+    def update(self, sql, param=[]):
+        self.db._cursor.execute(sql, param)
+        self.db._cursor.commit()
+        return True
+
+    def update_return_rowcount(self, sql, param=[]):
+        self.db._cursor.execute(sql, param)
+        n = self.db._cursor.rowcount
+        self.db._cursor.commit()
+        return n
+
+    def update_r_dict(self, sql, param=[]):
+        self.db._cursor.execute(sql, param)
+        result = self.db._cursor.fetchone()
+        if result:
+            result_cols = [column[0] for column in self.db._cursor.description]
+            result = dict(zip(result_cols, result))
+        self.db._cursor.commit()
+        return result
+
+    def update_r_list(self, sql, param=[]):
+        self.db._cursor.execute(sql, param)
+        result = self.db._cursor.fetchall()
+        result_cols = [column[0] for column in self.db._cursor.description]
+        self.db._cursor.commit()
+        return result, result_cols
+
+
+class Query:
+    pass
+

pynop-0.1.0/pynop/core/exceptions.py

@@ -0,0 +1,3 @@
+class BizError(Exception):
+    pass
+

pynop-0.1.0/pynop/core/log.py

@@ -0,0 +1,145 @@
+import logging
+from logging.handlers import TimedRotatingFileHandler
+from datetime import datetime
+import os
+import pickle
+import settings
+
+BIG_DIVIDER = """
+======================================================================
+"""
+
+DIVIDER = """
+----------------------------------------------------------------------
+"""
+
+BIG_LINEFEED = "\n\n"
+LINEFEED = "\n"
+
+
+class DataLogger:
+	def __init__(self, biz_name='DataBiz', log_level='INFO', log_dir_path='./', app_started_time=datetime.now()):
+		
+		if not os.path.exists(log_dir_path):
+			os.makedirs(log_dir_path)
+
+		self.log_level = log_level.upper()
+		self.log_dir_path = log_dir_path
+		self.app_started_time = app_started_time
+
+		logging.basicConfig(
+			#level=logging.DEBUG,
+		    #format="%(asctime)s [%(biz_name)-2s] %(levelname)-2s: %(message)s",
+		)
+		
+		self.logger = logging.getLogger(biz_name)
+		
+		if self.log_level == 'DEBUG':
+			self.logger.setLevel(logging.DEBUG)
+		elif self.log_level == 'INFO':
+			self.logger.setLevel(logging.INFO)
+		elif self.log_level == 'WARNING':
+			self.logger.setLevel(logging.WARNING)
+		elif self.log_level == 'ERROR':
+			self.logger.setLevel(logging.ERROR)
+		elif self.log_level == 'FATAL' or log_level == 'CRITICAL':
+			self.logger.setLevel(logging.CRITICAL)
+		else:
+			raise ValueError('unknown logging level')
+
+		log_filename = self.app_started_time.strftime("%Y%m%d_%H%M%S") + "_" + biz_name + ".log"
+		self.log_filename = os.path.join(self.log_dir_path, log_filename)
+
+		trh = TimedRotatingFileHandler(self.log_filename, when='D', interval=1, backupCount=settings.LOG_MAX_RETENTION_DAYS, encoding="utf8")
+		trh.suffix = "%Y%m%d_%H%M%S"
+		self.logger.addHandler(trh)
+		
+		ABC_SIGNATURE = f"""
+======================================================================
+	*Task Title : ABC Sync -> {biz_name}
+	*Started Time : {app_started_time}
+======================================================================
+"""
+		self.log(ABC_SIGNATURE, log_level)
+
+	def end(self):
+		handlers = self.logger.handlers[:]
+		for handler in handlers:
+			handler.close()
+			self.logger.removeHandler(handler)
+
+	def log(self, msg, log_level="warning"):		
+		if log_level.lower() == "warning":
+			self.logger.warning(msg)
+		elif log_level.lower() == "info":
+			self.logger.info(msg)
+		elif log_level.lower() == "debug":
+			self.logger.debug(msg)
+		elif log_level.lower() == "error":
+			self.logger.error(msg)
+
+	def log_big_divider(self, log_level="warning"):
+		self.log(BIG_DIVIDER, log_level)
+
+	def log_divider(self, log_level="warning"):
+		self.log(DIVIDER, log_level)
+
+	def log_big_linefeed(self, log_level="warning"):
+		self.log(BIG_LINEFEED, log_level)
+				
+	def log_linefeed(self, log_level="warning"):
+		self.log(LINEFEED, log_level)
+
+	def log_task_title(self, title="", log_level="warning"):
+		if not title:
+			title = self.logger.name
+		TITLE_SPACES = "	  "
+		msg = BIG_DIVIDER + LINEFEED + TITLE_SPACES + title + DIVIDER + LINEFEED
+		self.log(msg, log_level)
+
+	def log_func_title(self, title="", log_level="info"):		
+		msg = LINEFEED + LINEFEED + "- " + title + DIVIDER
+		self.log(msg, log_level)
+
+	def log_current_time(self):
+		self.log(LINEFEED + "** Started Time : {s}".format(s = self.app_started_time))
+		self.log(LINEFEED + "** Current Time : {c}".format(c = datetime.now()))
+		self.log(LINEFEED + "**  Wasted Time : {w}".format(w = datetime.now() - self.app_started_time))
+
+	def log_step(self, step="", log_level="info"):
+		msg = LINEFEED + BIG_DIVIDER + LINEFEED + ".. " + step + LINEFEED
+		self.log(msg, log_level)
+
+	def log_data(self, data={}, log_level="debug"):
+		for k, v in data.items():			
+			self.log(f"{k} : {v}", log_level)
+
+	def log_biz_msg(self, msg_title, msg_text, log_level="info"):
+		msg = LINEFEED + "*" + msg_title + ":" + LINEFEED + "  " + msg_text
+		self.log(msg, log_level)
+
+	def log_query(self, sql_name, sql):
+		self.log_divider("debug")
+		self.log_biz_msg(sql_name + "=\n", sql, "debug")
+
+	def log_query_result(self, sql_name, result):
+		self.log_biz_msg(sql_name + "_RESULT", str(result) + " rows", "debug")
+		self.log_divider("debug")
+
+	def dump_data(self, filename, data):
+		with open(filename, 'wb') as filehandle:
+			pickle.dump(data, filehandle, protocol=pickle.HIGHEST_PROTOCOL)
+
+	def load_data(self, filename):
+		with open(filename, 'rb') as filehandle:
+			return pickle.load(filehandle)
+
+	def cleanup_expired_log(self, log_dir, max_retention_days):
+		today = datetime.now()
+		for filename in os.listdir(log_dir):
+			file_path = os.path.join(log_dir, filename)
+			if os.path.isfile(file_path):
+				file_creation_time = datetime.fromtimestamp(os.path.getmtime(file_path))
+				if (today - file_creation_time).days > max_retention_days:
+					os.remove(file_path)
+

pynop-0.1.0/pynop/core/notifier.py

@@ -0,0 +1,157 @@
+import smtplib
+import settings
+from datetime import datetime
+import pymsteams
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from slack_sdk import WebClient
+from slack_sdk.errors import SlackApiError
+
+
+class Notify:
+    def __init__(self, name):
+        """
+        Initialize Notifier，support multiple notification tools.
+        :param name: notifier name
+        """
+        self.name = name
+        self.notifiers = self._initialize_notifiers()
+
+    def _initialize_notifiers(self):
+        # Automatically initializes all available notification tools.
+        return [
+            SlackNotify(self.name),
+            EmailNotify(self.name),
+        ]
+
+    def notify(self, subject, content):
+        # Call the notification method of all notification tools.
+        for notifier in self.notifiers:
+            try:
+                if hasattr(notifier, "notify"):
+                    notifier.notify(subject, content)
+            except Exception as e:
+                print(f"Error in {notifier.__class__.__name__} notification: {e}")
+
+    def notify_started(self):
+        self.notify("Started", "Batch task started successfully.")
+
+    def notify_exited(self):
+        self.notify("Exited", "Batch task exited successfully.")
+
+    def notify_error(self, tb):
+        self.notify("Has some error", "Error details are below:\n" + tb)
+
+    def notify_report(self, report_dict):
+        content = ""
+        for report_name, report_content in report_dict.items():
+            content += f"{report_name}: {str(report_content)}\n"
+        self.notify("Biz Reports", content.strip())
+
+    def close(self):
+        # Close all notification tool connections.
+        for notifier in self.notifiers:
+            try:
+                if hasattr(notifier, "close"):
+                    notifier.close()
+            except Exception as e:
+                print(f"Error closing {notifier.__class__.__name__} notifier: {e}")
+
+
+class SlackNotify:
+    def __init__(self, name):
+        self.name = name
+        try:
+            self.channel_id = settings.SLACK_CHANNEL_ID
+            if hasattr(settings, "SLACK_BOT_TOKEN") and settings.SLACK_BOT_TOKEN:
+                self.client = WebClient(token=settings.SLACK_BOT_TOKEN)
+            else:
+                self.client = None
+            print("slack setting ******")
+            print(self.channel_id, settings.SLACK_BOT_TOKEN)
+        except Exception as e:
+            print(f"Error instantiates Slack client: {e}")
+            self.client = None  # Make sure server is None for subsequent checks
+
+    def now(self):
+        return datetime.now().strftime("%Y-%m-%d %H:%M")
+
+    def notify(self, subject, content):
+        if self.client is None:
+            print("Slack client is not initialized.")
+            return
+        try:
+            result = self.client.chat_postMessage(
+                channel=self.channel_id,
+                text="*[{name}]* ({time}) {subject}\n{content}".format(
+                    name=self.name, time=self.now(), subject=subject, content=content
+                ),
+            )
+            if not result["ok"]:
+                print(result)
+
+        except SlackApiError as e:
+            print(f"Failed to send Slack message: {e}")
+
+
+class EmailNotify:
+    def __init__(self, name):
+        self.name = name
+        self.server = None  # Initializing the SMTP connection
+        try:
+            # connect to SMTP server
+            self.sender_email = settings.SENDER_EMAIL
+            self.receiver_email = settings.RECEIVER_EMAIL.split(",")
+            self.email_server_addr = settings.EMAIL_SERVER_ADDR
+            self.email_server_port = settings.EMAIL_SERVER_PORT
+
+            if self.email_server_addr and self.email_server_port and self.sender_email:
+                self.server = smtplib.SMTP(self.email_server_addr, self.email_server_port)
+        except Exception as e:
+            print(f"Error initializing SMTP connection: {e}")
+            self.server = None  # Make sure server is None for subsequent checks
+
+    def now(self):
+        return datetime.now().strftime("%Y-%m-%d %H:%M")
+
+    def notify(self, subject, content):
+        # Ban Success Mail
+        if subject != "Has some error":
+            return
+        if self.server is None:
+            print("SMTP server is not initialized.")
+            return
+
+        # Creat message object
+        message = MIMEMultipart()
+        message["From"] = self.sender_email
+        message["To"] = ", ".join(self.receiver_email)
+        message["Subject"] = "[{name}]{subject}".format(name=self.name, subject=subject)
+
+        # email text
+        msg = """
+<!DOCTYPE html>
+<html>
+<body>
+    <p><b>[{name}]({time}) {subject}</b></p>
+    <p>{content}</p>
+</body>
+</html>
+""".format(name=self.name, time=self.now(), subject=subject, content=content)
+        message.attach(MIMEText(msg, "html"))
+
+        try:
+            if self.sender_email and self.receiver_email:
+                # send email
+                self.server.sendmail(self.sender_email, self.receiver_email, message.as_string())
+        except Exception as e:
+            print(f"Failed to send email: {e}")
+
+    def close(self):
+        # Closing the SMTP Connection
+        if self.server is not None:
+            try:
+                self.server.quit()
+            except Exception as e:
+                print(f"Error closing SMTP connection: {e}")
+