re-common 2.0.1__py3-none-any.whl → 10.0.0__py3-none-any.whl

re_common/baselibrary/__init__.py

@@ -0,0 +1,4 @@
+from .readconfig.ini_config import IniConfig
+from .utils.mylogger import MLogger
+
+__all__ = ["MLogger", "IniConfig"]

re_common/baselibrary/baseabs/__init__.py

@@ -0,0 +1,7 @@
+from .baseabs import BaseAbs
+
+from ..readconfig.ini_config import IniConfig
+from ..utils.mylogger import MLogger
+from ..database.mbuilder import MysqlBuilderAbstract
+
+__all__ = ["BaseAbs", "MysqlBuilderAbstract", "IniConfig", "MLogger"]

re_common/baselibrary/baseabs/baseabs.py

@@ -0,0 +1,26 @@
+
+
+class BaseAbs(object):
+    """
+    抽象工厂模式，方便管理和引用 目前主要将配置文件和sql相关的库引入了抽象工厂模式中
+    """
+    @staticmethod
+    def get_sql_factory():
+        """
+        sql相关的工厂
+        :return:
+        """
+        from re_common.baselibrary.database.sql_factory import SqlFactory
+        return SqlFactory()
+
+    @staticmethod
+    def get_config_factory():
+        """
+        配置文件相关的工厂
+        :return:
+        """
+        from re_common.baselibrary.readconfig.config_factory import ConfigFactory
+        return ConfigFactory()
+
+
+

re_common/baselibrary/database/mbuilder.py

@@ -0,0 +1,132 @@
+from abc import ABCMeta, abstractmethod
+
+from re_common.baselibrary.database.moudle import SqlMoudle, Sqlite3Moudle
+
+
+class MysqlBuilderAbstract:
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def build_server_address(self, host):
+        pass
+
+    @abstractmethod
+    def build_username(self, user):
+        pass
+
+    @abstractmethod
+    def build_password(self, passwd):
+        pass
+
+    @abstractmethod
+    def build_port(self, port):
+        pass
+
+    @abstractmethod
+    def use_db(self, db):
+        pass
+
+    @abstractmethod
+    def build_chart(self, chartset):
+        pass
+
+    @abstractmethod
+    def get_moudle(self):
+        pass
+
+
+class Sqlite3BuilderAbstract:
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def build_file_path(self, dbpath):
+        pass
+
+    @abstractmethod
+    def build_isolation_level(self):
+        pass
+
+
+class MysqlBuilder(MysqlBuilderAbstract):
+    def __init__(self, configfile, configname, keytransformdicts=None):
+        self.sqlmoudle = SqlMoudle()
+
+        from re_common.baselibrary.readconfig.ini_config import IniConfig
+        self.ic = IniConfig(configfile).builder()
+        self.configfile = configfile
+        self.configname = configname
+        # 使用key转换器转换key
+        self.keytransformdicts = keytransformdicts
+        """
+        python2
+            self.config.readfp(open(configfile, 'rb'))
+        python3
+            self.config.read_file(open(configfile, 'r'))
+        """
+
+    def build_port(self, port):
+        self.sqlmoudle.port = int(self.ic.get_value(self.configname, port))
+
+    def build_server_address(self, host):
+        self.sqlmoudle.host = self.ic.get_value(self.configname, host)
+
+    def build_password(self, passwd):
+        self.sqlmoudle.passwd = self.ic.get_value(self.configname, passwd)
+
+    def use_db(self, db):
+        self.sqlmoudle.db = self.ic.get_value(self.configname, db)
+
+    def build_username(self, user):
+        self.sqlmoudle.user = self.ic.get_value(self.configname, user)
+
+    def build_chart(self, chartset):
+        self.sqlmoudle.charset = self.ic.get_value(self.configname, chartset)
+
+    def get_moudle(self):
+        return self.sqlmoudle
+
+    def build_all(self):
+        if not self.keytransformdicts:
+            self.keytransformdicts = {"port": "port", "host": "host", "passwd": "passwd", "user": "user", "db": "db",
+                                      "chartset": "chartset"}
+        self.build_port(self.keytransformdicts["port"])
+        self.build_server_address(self.keytransformdicts["host"])
+        self.build_password(self.keytransformdicts["passwd"])
+        self.build_username(self.keytransformdicts["user"])
+        self.use_db(self.keytransformdicts["db"])
+        self.build_chart(self.keytransformdicts["chartset"])
+        return self
+
+    def get_tuples(self):
+        return self.sqlmoudle.host, self.sqlmoudle.user, self.sqlmoudle.passwd, self.sqlmoudle.port
+
+
+class Sqlite3Builder(Sqlite3BuilderAbstract):
+    def __init__(self, configfile="", sec=""):
+        self.sqlite3moudle = Sqlite3Moudle()
+        from re_common.baselibrary.readconfig.ini_config import IniConfig
+        self.ic = IniConfig(configfile).builder()
+        self.configfile = configfile
+        self.sec = sec
+
+    def build_file_path(self, dbpath='', opt="dbpath"):
+        if dbpath:
+            self.sqlite3moudle.database = dbpath
+        else:
+            self.sqlite3moudle.database = self.ic.get_value(self.sec, opt)
+        return self
+
+    def build_timeout(self, timeout):
+        self.sqlite3moudle.timeout = timeout
+        return self
+
+    def build_isolation_level(self):
+        pass
+
+    def build_all(self):
+        self.build_file_path()
+        self.build_isolation_level()
+        return self
+
+    def get_moudle(self):
+        return self.sqlite3moudle

re_common/baselibrary/database/moudle.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+from pymysql.cursors import Cursor
+
+
+# pymysql的moudle
+class SqlMoudle(object):
+    def __init__(self):
+        """
+           host
+          string, host to connect
+          
+        user
+          string, user to connect as
+
+        passwd
+          string, password to use
+
+        db
+          string, database to use
+
+        port
+          integer, TCP/IP port to connect to
+
+        这是一个标准连接的所有参数
+        cursorclass pymysql.cursors.DictCursor 表示返回数据为dicts
+        """
+        self.host = None
+        self.user = None
+        self.password = ""
+        self.database = None
+        self.port = 3306
+        self.unix_socket = None
+        self.charset = ''
+        self.sql_mode = None
+        self.read_default_file = None
+        self.conv = None
+        self.use_unicode = True
+        self.client_flag = 0
+        self.cursorclass = Cursor
+        self.init_command = None
+        self.connect_timeout = 10
+        self.ssl = None
+        self.read_default_group = None
+        self.compress = None
+        self.named_pipe = None
+        self.autocommit = False
+        self.db = None
+        self.passwd = None
+        self.local_infile = False
+        self.max_allowed_packet = 16 * 1024 * 1024
+        self.defer_connect = False
+        self.auth_plugin_map = None
+        self.read_timeout = None
+        self.write_timeout = None
+        self.bind_address = None
+        self.binary_prefix = False
+        self.program_name = None
+        self.server_public_key = None
+
+    # @property
+    # def get_serveraddress(self):
+    #     return self._serveraddress
+    #
+    # @get_serveraddress.setter
+    # def set_serveraddress(self,serveraddress):
+    #     self._serveraddress = serveraddress
+    #
+    # @get_serveraddress.deleter
+    # def del_serveraddress(self):
+    #     del self._serveraddress
+
+    def to_dict(self):
+        return self.__dict__
+
+    def __str__(self):
+        return str(self.__dict__)
+
+
+# sqlite3的 moudle
+class Sqlite3Moudle(object):
+    def __init__(self):
+        self.database = ""
+        self.timeout = None
+        self.detect_types = None
+        self.isolation_level = None
+        self.check_same_thread = None
+        self.factory = None
+        self.cached_statements = None
+        # self.uri = None
+
+    def to_dict(self):
+        return self.__dict__

re_common/baselibrary/database/msqlite3.py

@@ -0,0 +1,194 @@
+import sqlite3
+
+from re_common.baselibrary.database.moudle import Sqlite3Moudle
+from re_common.baselibrary.database.sql_factory import SqlFactory
+from re_common.baselibrary.utils.basedict import BaseDicts
+
+
+def dict_factory(cursor, row):
+    """
+    设置返回字典
+    :param row:
+    :return:
+    """
+    d = {}
+    for index, col in enumerate(cursor.description):
+        d[col[0]] = row[index]
+    return d
+
+
+class Sqlite3(SqlFactory):
+    def __init__(self):
+        # 就是conn
+        self.db = ''
+        self.cursor = ''
+        self.sqlite3_moudle = {}
+
+    def set_result_dict(self):
+        def dict_factory(cursor, row):
+            d = {}
+            for idx, col in enumerate(cursor.description):
+                d[col[0]] = row[idx]
+            return d
+
+        self.db.row_factory = dict_factory
+        self.get_cursor()
+        return self
+
+    def set_encoding(self, bytes=False, encoding="gbk"):
+        """
+        https://blog.csdn.net/xkxjy/article/details/8179479
+        在不知道编码格式的情况下 使用bytes 但我没有成功
+        :param bytes:
+        :param encoding:
+        :return:
+        """
+        # if bytes:
+        #     self.db.text_factory = bytes
+        # else:
+        #     self.db.text_factory = lambda x: str(x, encoding, 'ignore')
+        self.db.text_factory = lambda x: str(x, encoding, 'ignore')
+
+    def link(self, sqlite3_moudle: Sqlite3Moudle):
+        """
+        连接数据库
+        :param sqlite3_moudle:
+        :param mysqlmoudle:
+        :return:
+        """
+        self.sqlite3_moudle = sqlite3_moudle
+        # 返回连接对象
+        dicts = sqlite3_moudle.to_dict()
+        dicts = BaseDicts.removeDictsNone(dicts)
+        self.db = sqlite3.connect(**dicts)
+        self.cursor = self.db.cursor()
+        return self.db, self.cursor
+
+    def get_cursor(self) -> sqlite3.Cursor:
+        """
+        本函数通过链接返回sqlite3的游标
+        :param connect: 传入的是sqlite3的连接
+        :return: 返回游标
+        """
+        assert isinstance(self.db, sqlite3.Connection)
+        self.cursor = self.db.cursor()
+        return self.cursor
+
+    def execute(self, sql: str) -> sqlite3.Cursor:
+        """
+        通过游标对象执行sql语句并返回结果
+        :param cursor: 游标对象
+        :param sql:  需要执行的sql语句
+        :return:  返回结果的游标对象
+        """
+        assert isinstance(self.cursor, sqlite3.Cursor)
+        result = self.cursor.execute(sql)
+        return result
+
+    def executemany(self, sql: str, seq_of_parameters) -> sqlite3.Cursor:
+        assert isinstance(self.cursor, sqlite3.Cursor)
+        result = self.cursor.executemany(sql, seq_of_parameters)
+        return result
+
+    def get_all_field_info(self, tablename):
+        """
+        获取字段详细信息
+        :param tablename:
+        :return:
+        """
+        sql = "PRAGMA table_info({})".format(tablename)
+        result = self.execute(sql)
+        return result.fetchall()
+
+    def get_all_field(self, tablename):
+        """
+        仅仅获取字段
+        :param tablename:
+        :return:
+        """
+        listinfo = self.get_all_field_info(tablename)
+        return [row[1] for row in listinfo]
+
+    def get_table_name(self):
+        """
+        获取表名
+        :return:
+        """
+        sql = "select name from sqlite_master where type='table' order by name;"
+        result = self.execute(sql)
+        return result.fetchall()
+
+    def get_rowcount(self, ) -> int:
+        """
+        获取sql语句影响的行数
+        :param cursor: 游标对象
+        :return:  影响的行数
+        """
+        assert isinstance(self.cursor, sqlite3.Cursor)
+        num = self.cursor.rowcount
+        return num
+
+    def close_cursor(self) -> None:
+        """
+        关闭游标
+        :param cursor: 游标
+        :return: None
+        """
+        assert isinstance(self.cursor, sqlite3.Cursor)
+        self.cursor.close()
+
+    def my_commit(self) -> None:
+        """
+        提交
+        :param connect: 连接
+        :return: None
+        """
+        assert isinstance(self.db, sqlite3.Connection)
+        self.db.commit()
+
+    def close_connect(self) -> None:
+        """
+        链接关闭
+        :param connect: 连接
+        :return: None
+        """
+        assert isinstance(self.db, sqlite3.Connection)
+        self.db.close()
+
+    def __del__(self):
+        self.close_connect()
+
+    @classmethod
+    def sqlite3_merge(cls, inpath, attachpath, tablename="modify_title_info_zt"):
+        """
+        合并两个db3
+        :param inpath:
+        :param attachpath:
+        :return:
+        """
+        conn = sqlite3.connect(inpath)
+        conn.text_factory = str
+        cur = conn.cursor()
+        attach = 'attach database "' + attachpath + '" as w;'
+        sql1 = 'insert into {} select * from w.{};'.format(tablename, tablename)
+        cur.execute(attach)
+        cur.execute(sql1)
+        conn.commit()
+        cur.close()
+        conn.close()
+
+    def create_table(self, tablename, fields: list):
+        if self.cursor == '':
+            self.get_cursor()
+        sql = "PRAGMA foreign_keys = false;"
+        self.execute(sql)
+        sql = f'DROP TABLE IF EXISTS "{tablename}";'
+        self.execute(sql)
+        strings = ""
+        for field in fields:
+            strings = strings + f'"{field}" TEXT,'
+        strings = strings.rstrip(',')
+        sql = f'CREATE TABLE "{tablename}"({strings});'
+        self.execute(sql)
+        sql = 'PRAGMA foreign_keys = true;'
+        self.execute(sql)

re_common/baselibrary/database/mysql.py

@@ -0,0 +1,169 @@
+import pymysql
+
+from re_common.baselibrary.database.moudle import SqlMoudle
+from re_common.baselibrary.database.sql_factory import SqlFactory
+from re_common.baselibrary.utils.version_compare import compare_bool
+
+_json_escape_table = [chr(x) for x in range(128)]
+_json_escape_table[0] = "\\0"
+_json_escape_table[ord("\\")] = "\\\\"
+_json_escape_table[ord("\n")] = "\\n"
+_json_escape_table[ord("\r")] = "\\r"
+_json_escape_table[ord("\032")] = "\\Z"
+_json_escape_table[ord('"')] = '\\"'
+_json_escape_table[ord("'")] = "\'"
+
+
+class Mysql(SqlFactory):
+
+    @property
+    def db(self):
+        # 就是conn
+        if not self.is_ping():
+            assert self._mysqlmoudle, AttributeError("MySQL model不存在 无法重新连接 请设置moudle")
+            self.reConnect()
+        return self._db  # type: pymysql.cursors.Cursor
+
+    @db.setter
+    def db(self, value):
+        self._db = value
+
+    @property
+    def cursor(self):
+        if self._cursor.connection:
+            return self._cursor  # type: pymysql.cursors.Cursor
+        else:
+            self._cursor = self._db.cursor()
+            return self._cursor
+
+    @cursor.setter
+    def cursor(self, value):
+        self._cursor = value
+
+    @property
+    def mysqlmoudle(self):
+        return self._mysqlmoudle  # type: SqlMoudle
+
+    @mysqlmoudle.setter
+    def mysqlmoudle(self, moudle):
+        self._mysqlmoudle = moudle
+
+    def get_new_cursor(self):
+        """
+        获取一个新的游标
+        :return:
+        """
+        # 检查db的存在及是否断掉 不知道为什么 不可以加括号 但编辑器会警告
+        return self._db.cursor()
+
+    def link(self, mysqlmoudle: SqlMoudle):
+        """
+        连接数据库
+        :param mysqlmoudle:
+        :return:
+        """
+        self._mysqlmoudle = mysqlmoudle
+        # 返回连接对象
+        self._db = pymysql.connect(**mysqlmoudle.to_dict())
+        self._cursor = self._db.cursor()
+
+        return self
+
+    def reConnect(self):
+        """
+        重新连接数据库
+        :return:
+        """
+        try:
+            self._db.ping()
+            self._db.close()
+        except:
+            self.link(self._mysqlmoudle)
+        self.link(self._mysqlmoudle)
+
+    def is_ping(self):
+        if self._db:
+            try:
+                return self._db.ping()
+            except:
+                self.link(self._mysqlmoudle)
+        else:
+            raise AttributeError("数据库连接对象不存在，请调用reConnect重新连接")
+
+    def commit(self):
+        """
+        事务提交
+        :return:
+        """
+        # 对象判断
+        assert isinstance(self._db, pymysql.connections.Connection)
+        self._db.commit()
+
+    def execute(self, sql, args=None):
+        """
+        执行sql语句
+        :param sql:
+        :return:
+        """
+        try:
+            assert isinstance(self._cursor, pymysql.cursors.Cursor)
+            # 游标是否被关闭,执行该函数保证有游标
+            self._cursor.execute(sql, args=args)
+            self.commit()
+        except Exception as e:
+            raise e
+
+    def fetchall(self):
+        """
+        select 后调用获取全部返回结果
+        :return:
+        """
+        result = self._cursor.fetchall()
+        return result
+
+    def close(self):
+        if self._db and isinstance(self._db, pymysql.connections.Connection):
+            self._db.close()
+
+    def rollback(self):
+        self.db.rollback()
+
+    @classmethod
+    def escape(cls, strings):
+        # 这个不能用于多线程，否则会出现多线程抢占连接
+        if compare_bool(pymysql.__version__, "1.0.0"):
+            from pymysql.converters import escape_string
+            return escape_string(strings)
+        else:
+            from pymysql import escape_string
+            return escape_string(strings)
+
+    def json_escape(self, strings):
+        """
+        json化需要
+        :param strings:
+        :return:
+        """
+        return strings.translate(_json_escape_table)
+
+    def __repr__(self):
+        return 'Mysql().link(%s)' % self._mysqlmoudle
+
+    def __str__(self):
+        return "Mysql %s" % self._mysqlmoudle
+
+    def __del__(self):
+        self.close()
+
+
+def json_update(dicts):
+    duplicte_list = []
+    for k, v in dicts.items():
+        if isinstance(v, str):
+            duplicte_list.append(f"'$.{Mysql.escape(k)}','{Mysql.escape(v)}'")
+        else:
+            duplicte_list.append(f"'$.{Mysql.escape(k)}','{v}'")
+    duplicte = ",".join(duplicte_list)
+    # 处理 符合 %s 占位符里面的 正常数据非占位符存在 % 的情况
+    duplicte_format = duplicte.replace("%", "%%")
+    return duplicte_format

re_common/baselibrary/database/sql_factory.py

@@ -0,0 +1,26 @@
+from re_common.baselibrary.baseabs import BaseAbs
+
+
+class SqlFactory(BaseAbs):
+    """
+    sql 工厂类
+    """
+    @staticmethod
+    def mysql_factory(type='mysql'):
+        if type == 'mysql':
+            """
+            mysql类 返回一个mysql的对象
+            """
+            from re_common.baselibrary.database.mysql import Mysql
+            return Mysql()
+        assert 0, "err sql type please check: %s" % type
+
+    @staticmethod
+    def sqlite_factory(type='sqlite3'):
+        if type == 'sqlite3':
+            """
+             sqlite3类 返回一个sqlite3的对象
+             """
+            from re_common.baselibrary.database.msqlite3 import Sqlite3
+            return Sqlite3()
+        assert 0, "err sqllite type please check: %s" % type