icemammoth-common 0.1.0.dev0__tar.gz

icemammoth_common-0.1.0.dev0/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.1
+Name: icemammoth_common
+Version: 0.1.0.dev0
+Summary: common tools collection
+Home-page: 
+Author: Klein
+Author-email: myicemammoth@gmail.com
+Keywords: common utils
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.12
+Description-Content-Type: text/markdown
+Requires-Dist: beautifulsoup4>=4.12.3
+Requires-Dist: coloredlogs>=15.0.1
+Requires-Dist: mysql_connector_repackaged>=0.3.1
+Requires-Dist: Pillow>=10.3.0
+Requires-Dist: Requests>=2.32.2

icemammoth_common-0.1.0.dev0/icemammoth_common/base/__init__.py

@@ -0,0 +1 @@
+from .base_object import *

icemammoth_common-0.1.0.dev0/icemammoth_common/base/base_object.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+class BaseObject(object):
+    
+    def __repr__(self) -> str:
+        return self.__str__()
+    
+    def __str__(self) -> str:
+        attributes = [
+            attribute
+            for attribute in dir(self)
+            if not attribute.startswith("__") and not callable(getattr(self, attribute))
+        ]
+        attributeMap = {
+            attribute: str(getattr(self, attribute)) for attribute in attributes
+        }
+        return f"{attributeMap}" 

icemammoth_common-0.1.0.dev0/icemammoth_common/base/singleton.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+class SingletonMeta(type):
+    """
+    The Singleton class can be implemented in different ways in Python. Some
+    possible methods include: base class, decorator, metaclass. We will use the
+    metaclass because it is best suited for this purpose.
+    """
+
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        """
+        Possible changes to the value of the `__init__` argument do not affect
+        the returned instance.
+        """
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]

icemammoth_common-0.1.0.dev0/icemammoth_common/database/base_dao.py

@@ -0,0 +1,320 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any, Dict, List, Tuple, TypeVar, Generic
+from base import BaseObject
+
+from database.connect_pool import ConnectPool
+from .transaction import TransactionManager
+from icemammoth_common.util.log_util import logger
+from icemammoth_common.util import string_util, object_util
+
+class ModelAttrValueConvertor(object):
+
+    def attrValueToColumnValue(self, table, attrName, attrValue):
+        pass
+
+    def columnValueToAttrValue(self, table, columnName, columnValue):
+        pass
+
+
+class DAOInheritException(BaseException):
+
+    def __init__(self, msg):
+        super(msg)
+
+
+class ORMConfig(BaseObject):
+
+    def __init__(
+        self,
+        table: str = None,
+        # columns: [str] = None,
+        model=None,
+        columnAttrMap: Dict[str, str] = None,
+        attrValueConvertor: ModelAttrValueConvertor = None,
+    ):
+        self.table = table
+        self.model = model
+        self.columnAttrMap = columnAttrMap
+        self.attrValueConvertor = attrValueConvertor
+
+
+# 使用泛型,必须先使用TypeVar定义泛型类型
+ModelType = TypeVar("ModelType", bound=BaseObject)
+
+def defaultProcessFunc(cursor) :
+    return cursor.fetchall()
+
+
+class BaseDAO(Generic[ModelType]):
+
+    def __init__(self):
+        if not self.__ORM_CONFIG__:
+            raise DAOInheritException(
+                f"class {
+                    self.__class__.__name__} inherit from BaseDAO must set attribute __ORM_CONFIG__"
+            )
+        self.ormConfig = self.__ORM_CONFIG__
+        self.table = self.ormConfig.table
+        self.modelType = self.ormConfig.model
+        self.modelAttrs = object_util.fetchObjectAttributes(
+            self.ormConfig.model())
+        self.attrColumnMap = {}
+        self.columnAttrMap = {}
+        self.mappedColumns = []
+        self.mappedAttrs:List[str] = []
+        for column, attr in self.ormConfig.columnAttrMap.items():
+            if not attr:
+                attr = string_util.underscore_to_camel(column)
+            if attr not in self.modelAttrs:
+                raise NotExistError(
+                    f"dao {self.__class__.__name__} orm config error, class {self.ormConfig.model.__name__} doesn't exist attribute {
+                        attr} which is mapped with column {self.table}.{column}"
+                )
+            self.attrColumnMap[attr] = column
+            self.columnAttrMap[column] = attr
+            self.mappedColumns.append(column)
+            self.mappedAttrs.append(attr)
+
+        self.attrValueConvertor = self.ormConfig.attrValueConvertor
+        self.connectionPool = ConnectPool()
+        self.transactionManager = TransactionManager()
+
+    def _fetchValuesByAttrs(self, model, attrs=None) -> List[Any]:
+        row = []
+
+        attrValueMap = object_util.getObjectAttributesAndValues(model)
+        attrs = attrs if attrs else self.mappedAttrs
+
+        for attr in attrs:
+            value = attrValueMap.get(attr)
+            value = self.attrValueConvertor.attrValueToColumnValue(
+                self.table, attr, value) if self.attrValueConvertor else value
+            row.append(value)
+
+        return row
+    def fillRowDataToModel(self, columns: List[str], row: List[Any], model):
+
+        if row is None or columns is None:
+            return None
+
+        for idx, column in enumerate(columns):
+            attr = self._fetchAttrByColum(column)
+            if not attr:
+                raise NotExistError(
+                    f"dao {self.__class__.__name__} convert row to model failed, class {
+                        self.ormConfig.model.__name__} no attribute mapped with column {self.table}.{column}."
+                )
+            value = row[idx]
+            value = self.attrValueConvertor.columnValueToAttrValue(
+                self.table, column, value) if self.attrValueConvertor else value
+            setattr(model, attr, value)
+        return model
+
+    def _insertData(self, cursor):
+        return cursor.lastrowid
+
+    def _insertDatas(self, cursor):
+        return (cursor.lastrowid, cursor.rowcount)
+
+    def _readDatas(self, cursor):
+        return (
+            [
+                self.fillRowDataToModel(
+                    cursor.column_names, result, self.ormConfig.model())
+                for result in results
+            ]
+            if (results := cursor.fetchall())
+            else None
+        )
+
+    def _readData(self, cursor):
+        result = cursor.fetchone()
+        return self.fillRowDataToModel(cursor.column_names, result, self.ormConfig.model())
+
+    def _deleteDatas(self, cursor):
+        return cursor.rowcount
+
+    def _updateDatas(self, cursor):
+        return cursor.rowcount
+
+    def execute(self, sql: str, params:List[Any], process_func=None, executeMany=False):
+        try:
+            connection = None
+            if self.transactionManager.inTransaction:
+                connection = self.transactionManager.connection
+            else:
+                connection = self.connectionPool.getConnection()
+            cursor = connection.cursor()
+            if executeMany:
+                cursor.executemany(sql, params)
+            else:
+                cursor.execute(sql, params)
+        
+            result = None
+            if process_func is None:
+                result = defaultProcessFunc(cursor)
+            else:
+                result = process_func(cursor)
+            cursor.close
+            return result
+        except Exception as e:
+            logger.exception(
+                f"execute sql failed!sql:{sql},params:{params},executeMany:{executeMany}"
+            )
+            raise e
+        finally:
+            if not self.transactionManager.inTransaction and connection:
+                connection.close()
+
+    def insertData(self, model: ModelType, ignoreDuplicate=False, duplicateUpdateAttrs: List[str] = []) -> int:
+        sql: str = self._generateInsertSql(ignoreDuplicate=ignoreDuplicate, duplicateUpdateAttrs=duplicateUpdateAttrs)
+        logger.debug(f"sql is {sql}")
+        params: List[Any] = self._fetchValuesByAttrs(model)
+        return self.execute(sql, params, self._insertData)
+
+    def insertDatas(self, models: List[ModelType], ignoreDuplicate=False, duplicateUpdateAttrs: List[str] = []) -> Tuple[int, int]:
+        sql: str = self._generateInsertSql(ignoreDuplicate=ignoreDuplicate, duplicateUpdateAttrs=duplicateUpdateAttrs)
+        logger.debug(f"sql is {sql}")
+        params: List[List[Any]] = [self._fetchValuesByAttrs(model) for model in models]
+        return self.execute(sql, params, self._insertDatas, True)
+    def _generateInsertSql(self, insertAttrs: List[str] = None, ignoreDuplicate=False, duplicateUpdateAttrs: List[str] = []):
+        columns = []
+        if insertAttrs:
+            for attr in insertAttrs:
+                column = self._fetchColumnByAttr(attr)
+                columns.append(column)
+        else:
+            columns = self.mappedColumns
+        columnsSQL = ",".join(columns)
+        ignoreDuplicateTag = 'IGNORE' if ignoreDuplicate else ''
+        valuesSQL = ','.join(['%s']*len(columns))
+
+        duplicateUpdateSQL: str = ""
+        if not ignoreDuplicate and duplicateUpdateAttrs:
+
+            duplicateUpdateSQL = 'ON DUPLICATE KEY UPDATE'
+            for attr in duplicateUpdateAttrs:
+                column = self._fetchColumnByAttr(attr)
+                duplicateUpdateSQL = f'{duplicateUpdateSQL} {column} = VALUES({column}), '
+
+            duplicateUpdateSQL = duplicateUpdateSQL[:-2]
+
+        return f"INSERT {ignoreDuplicateTag} INTO {self.table} ({columnsSQL}) VALUES ({valuesSQL}) {duplicateUpdateSQL}"
+
+    def readData(self, readAttrs: List[str] = None, **conditions) -> ModelType:
+        sql = self._generateReadSql(
+            readOne=True, readAttrs=readAttrs, **conditions)
+        params = list(conditions.values())
+        return self.execute(sql, params, self._readData)
+
+    def readDatas(self, readAttrs: List[str] = None, **conditions) -> List[ModelType]:
+        sql = self._generateReadSql(
+            readOne=False, readAttrs=readAttrs, **conditions)
+        params = list(conditions.values())
+        return self.execute(sql, params, self._readDatas)
+
+    def readDatasByConditionSQL(
+        self, readAttrs: List[str] = None, conditionSql: str = None, params=None
+    ) -> List[ModelType]:
+
+        readColumns = []
+        if readAttrs:
+            for attr in readAttrs:
+                column = self._fetchColumnByAttr(attr)
+                readColumns.append(column)
+        else:
+            readColumns = self.mappedColumns
+        columnsSQL: str = ",".join(readColumns)
+
+        selectSQL = f"SELECT {columnsSQL} FROM {
+            self.table} WHERE {conditionSql}"
+        return self.execute(selectSQL, params, self._readDatas)
+
+    def _generateReadSql(self, readOne: bool = False, readAttrs: List[str] = None, **conditions):
+
+        params = []
+
+        readColumns = []
+        if readAttrs:
+            for attr in readAttrs:
+                column = self._fetchColumnByAttr(attr)
+                readColumns.append(column)
+        else:
+            readColumns = self.mappedColumns
+        columnsSQL: str = ",".join(readColumns)
+
+        conditionColumns = []
+        for attr, value in conditions.items():
+            column = self._fetchColumnByAttr(attr)
+            conditionColumns.append(column)
+            value = self.attrValueConvertor.attrValueToColumnValue(
+                self.table, attr, value) if self.attrValueConvertor else value
+            params.append(value)
+        conditionSQL: str = " and ".join(
+            [f"{column} = %s" for column in conditionColumns])
+
+        limitSQL: str = "LIMIT 1" if readOne else ""
+
+        return f"SELECT {columnsSQL} FROM {self.table} WHERE {conditionSQL} {limitSQL}"
+
+    def deleteDatas(self, **conditions):
+
+        params = []
+
+        conditionColumns = []
+        for attr, value in conditions.items():
+            column = self._fetchColumnByAttr(attr)
+            conditionColumns.append(column)
+            value = self.attrValueConvertor.attrValueToColumnValue(
+                self.table, attr, value) if self.attrValueConvertor else value
+            params.append(value)
+        conditionSQL = " AND ".join(
+            [f"{column} = %s" for column in conditionColumns])
+
+        deleteSQL = f"DELETE FROM {self.table} WHERE {conditionSQL}"
+
+        return self.execute(deleteSQL, params, self._deleteDatas)
+
+    def updateDatas(self, updateAttrs: Dict[str, str], **conditions):
+
+        updateColumns = []
+        params = []
+
+        for attr, value in updateAttrs.items():
+            column = self._fetchColumnByAttr(attr)
+            updateColumns.append(column)
+            value = self.attrValueConvertor.attrValueToColumnValue(
+                self.table, attr, value) if self.attrValueConvertor else value
+            params.append(value)
+        updateColumnSQL = ', '.join(
+            [f'{column} = %s' for column in updateColumns])
+
+        conditionColumns = []
+        for attr, value in conditions.items():
+            column = self._fetchColumnByAttr(attr)
+            conditionColumns.append(column)
+            value = self.attrValueConvertor.attrValueToColumnValue(
+                self.table, attr, value) if self.attrValueConvertor else value
+            params.append(value)
+        conditionSQL = " AND ".join(
+            [f"{column} = %s" for column in conditionColumns])
+
+        updateSQL = f"UPDATE {self.table} SET {
+            updateColumnSQL} WHERE {conditionSQL}"
+
+        return self.execute(updateSQL, params, self._updateDatas)
+
+    def _fetchColumnByAttr(self, attr: str):
+        column = self.attrColumnMap.get(attr)
+        if not column:
+            raise TypeError(f'table {self.table} not column bind with attr {
+                            self.modelType.__qualname__}.{attr}')
+        return column
+
+    def _fetchAttrByColum(self, column):
+        attr = self.columnAttrMap.get(column)
+        if not attr:
+            raise TypeError(f'{self.modelType.__qualname__} not attribute bind with table {
+                            self.table} column {column}')
+        return attr

icemammoth_common-0.1.0.dev0/icemammoth_common/database/connect_pool.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+
+from mysql.connector import pooling
+from icemammoth_common.util import module_util
+from icemammoth_common.base.singleton import SingletonMeta
+from icemammoth_common.util import module_util
+from icemammoth_common.util import log_util
+
+CONFIG_MODULE_NAME = "config"
+
+class ConnectPool(metaclass=SingletonMeta):
+
+    def __init__(self):
+
+        if not module_util.module_exist(CONFIG_MODULE_NAME):
+            log_util.logger.exception(
+                "config module is not exist, must set config module first!")
+            raise Exception(
+                "config module is not exist, must set config module first!")
+
+        datebase_host = module_util.get_variable(
+            CONFIG_MODULE_NAME, "DATABASE_HOST", except_unexist=True)
+        database_port = module_util.get_variable(
+            CONFIG_MODULE_NAME, "DATABASE_PORT", except_unexist=True)
+        database_user = module_util.get_variable(
+            CONFIG_MODULE_NAME, "DATABASE_USER", except_unexist=True)
+        database_password = module_util.get_variable(
+            CONFIG_MODULE_NAME, "DATABASE_PASSWORD", except_unexist=True)
+        database_db_name = module_util.get_variable(
+            CONFIG_MODULE_NAME, "DATABASE_DB_NAME", except_unexist=True)
+        database_connect_pool_size = module_util.get_variable(
+            CONFIG_MODULE_NAME, "DATABASE_CONNECT_POOL_SIZE", except_unexist=True)
+
+        # dbconfig = {
+        #     "host": datebase_host,
+        #     "port": database_port,
+        #     "user": database_user,
+        #     "password": database_password,
+        #     "database":database_db_name,
+        # }
+
+        self.cnxpool = pooling.MySQLConnectionPool(host=datebase_host,
+                                                   port=database_port,
+                                                   user=database_user,
+                                                   password=database_password,
+                                                   database=database_db_name,
+                                                   pool_name="connection-pool",
+                                                   pool_reset_session=True,
+                                                   pool_size=database_connect_pool_size,
+                                                   autocommit=True)
+
+    def getConnection(self):
+        return self.cnxpool.get_connection()

icemammoth_common-0.1.0.dev0/icemammoth_common/database/transaction.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+
+import threading
+from icemammoth_common.decorator import decorator
+from icemammoth_common.util.log_util import logger
+from database.connect_pool import ConnectPool
+
+'''
+    ThreadLocal.local每次调用都会返回一个新对象,并非线程内唯一
+    作为全局变量方便实现线程的内变量共享,放到函数体内则在函数每次执行时会生成不同的变量,无法实现线程的内变量共享
+    这点与Java中的ThreadLocal相同
+'''
+local = threading.local()
+
+class TransactionManager(object):
+    
+    connection = None
+    inTransaction = False
+    
+    def __init__(self):
+        self.connectionPool = ConnectPool()
+    
+    def startTx(self):
+        TransactionManager.connection = self.connectionPool.getConnection()
+        TransactionManager.connection.start_transaction()
+        TransactionManager.inTransaction = True
+    
+    def commit(self):
+        try:
+            TransactionManager.connection.commit()
+            TransactionManager.connection.close()
+        finally:
+            TransactionManager.connection = None
+            TransactionManager.inTransaction = False
+    
+    def rollback(self):
+        try:
+            TransactionManager.connection.rollback()
+            TransactionManager.connection.close()
+        finally:
+            TransactionManager.connection = None
+            TransactionManager.inTransaction = False
+            
+def transaction():
+    @decorator
+    def transaction_decorator(f, *fargs, **fkwargs):
+        firstLevel = False
+        # transactionManager = None
+        try:
+            local.transactionManager
+        except AttributeError as error:
+            logger.debug('no transactionManager in thread local')
+            logger.debug('enter transaction scope')
+            local.transactionManager = None
+
+        if not local.transactionManager:
+            firstLevel = True
+            local.transactionManager = TransactionManager()
+            # transactionManager = local.transactionManager
+        
+        try:
+            if firstLevel:
+                logger.debug(msg='start transaction')
+                local.transactionManager.startTx()
+            result = f(*fargs,**fkwargs)
+            if firstLevel:
+                logger.debug('commit transaction')
+                local.transactionManager.commit()
+            return result
+        except Exception as exception:
+            if firstLevel and local.transactionManager.inTransaction:
+                logger.exception(f'transaction execute failed, rollback!func:{f.__module__}:{f.__qualname__}')
+                local.transactionManager.rollback()
+            raise exception
+        finally:
+            if firstLevel:
+                logger.debug('exit transaction scope')
+                local.transactionManager = None
+        
+    return transaction_decorator

icemammoth_common-0.1.0.dev0/icemammoth_common/decorator/retry.py

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+
+import random
+import time
+import traceback
+from icemammoth_common.util.log_util import logger as Logger
+from functools import partial
+
+from decorator import decorator
+
+
+def __retry_internal(
+    f,
+    exceptions=Exception,
+    tries=-1,
+    delay=0,
+    max_delay=10,
+    backoff=1,
+    jitter=0,
+    logger=Logger,
+    log_traceback=False,
+    on_exception=None,
+):
+    """
+    Executes a function and retries it if it failed.
+
+    :param f: the function to execute.
+    :param exceptions: an exception or a tuple of exceptions to catch. default: Exception.
+    :param tries: the maximum number of attempts. default: -1 (infinite).
+    :param delay: initial delay between attempts. default: 0.
+    :param max_delay: the maximum value of delay. default: None (no limit).
+    :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff).
+    :param jitter: extra seconds added to delay between attempts. default: 0.
+                   fixed if a number, random if a range tuple (min, max)
+    :param logger: logger.warning(fmt, error, delay) will be called on failed attempts.
+                   default: retry.logging_logger. if None, logging is disabled.
+    :param on_exception: handler called when exception occurs. will be passed the captured
+                         exception as an argument. further retries are stopped when handler
+                         returns True. default: None
+    :returns: the result of the f function.
+    """
+    _tries, _delay = tries, delay
+    while _tries:
+        try:
+            return f()
+        except exceptions as e:
+            if on_exception is not None and on_exception(e):
+                raise e
+
+            _tries -= 1
+            if _tries <= 0:
+                raise e
+
+            try:
+                func_qualname = f.func.__qualname__
+            except AttributeError:
+                func_qualname = str(f.func)
+            logger.warning(
+                f'method {f.func.__module__}.{func_qualname} execute failed, ' + \
+                f'will do {tries-_tries}th retry in {_delay} seconds. ' + \
+                f'error_type:{e.__class__.__qualname__}, error_message:{e}'
+            )
+
+            if log_traceback:
+                logger.warning(traceback.format_exc())
+
+            time.sleep(_delay)
+            _delay *= backoff
+
+            _delay += random.uniform(*jitter) if isinstance(jitter, tuple) else jitter
+            if max_delay is not None:
+                _delay = min(_delay, max_delay)
+
+
+def retry(
+    exceptions=Exception,
+    tries=-1,
+    delay=0,
+    max_delay=10,
+    backoff=1,
+    jitter=0,
+    logger=Logger,
+    log_traceback=False,
+    on_exception=None,
+):
+    """Returns a retry decorator.
+
+    :param exceptions: an exception or a tuple of exceptions to catch. default: Exception.
+    :param tries: the maximum number of attempts. default: -1 (infinite).
+    :param delay: initial delay between attempts. default: 0.
+    :param max_delay: the maximum value of delay. default: None (no limit).
+    :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff).
+    :param jitter: extra seconds added to delay between attempts. default: 0.
+                   fixed if a number, random if a range tuple (min, max)
+    :param logger: logger.warning(fmt, error, delay) will be called on failed attempts.
+                   default: retry.logging_logger. if None, logging is disabled.
+    :param on_exception: handler called when exception occurs. will be passed the captured
+                         exception as an argument. further retries are stopped when handler
+                         returns True. default: None
+    :returns: a retry decorator.
+    """
+
+    @decorator
+    def retry_decorator(f, *fargs, **fkwargs):
+        args = fargs or []
+        kwargs = fkwargs or {}
+        return __retry_internal(
+            partial(f, *args, **kwargs),
+            exceptions,
+            tries,
+            delay,
+            max_delay,
+            backoff,
+            jitter,
+            logger,
+            log_traceback,
+            on_exception,
+        )
+
+    return retry_decorator
+
+
+def retry_call(
+    f,
+    fargs=None,
+    fkwargs=None,
+    exceptions=Exception,
+    tries=-1,
+    delay=0,
+    max_delay=10,
+    backoff=1,
+    jitter=0,
+    logger=Logger,
+    log_traceback=False,
+    on_exception=None,
+):
+    """
+    Calls a function and re-executes it if it failed.
+
+    :param f: the function to execute.
+    :param fargs: the positional arguments of the function to execute.
+    :param fkwargs: the named arguments of the function to execute.
+    :param exceptions: an exception or a tuple of exceptions to catch. default: Exception.
+    :param tries: the maximum number of attempts. default: -1 (infinite).
+    :param delay: initial delay between attempts. default: 0.
+    :param max_delay: the maximum value of delay. default: None (no limit).
+    :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff).
+    :param jitter: extra seconds added to delay between attempts. default: 0.
+                   fixed if a number, random if a range tuple (min, max)
+    :param logger: logger.warning(fmt, error, delay) will be called on failed attempts.
+                   default: retry.logging_logger. if None, logging is disabled.
+    :param on_exception: handler called when exception occurs. will be passed the captured
+                         exception as an argument. further retries are stopped when handler
+                         returns True. default: None
+    :returns: the result of the f function.
+    """
+    args = fargs or []
+    kwargs = fkwargs or {}
+    return __retry_internal(
+        partial(f, *args, **kwargs),
+        exceptions,
+        tries,
+        delay,
+        max_delay,
+        backoff,
+        jitter,
+        logger,
+        log_traceback,
+        on_exception,
+    )