mdbq 1.5.5__py3-none-any.whl → 1.5.7__py3-none-any.whl

mdbq/config/get_myconf.py

@@ -21,7 +21,7 @@ class MyConf:
 
     def get_myconf(self, options: list):
         if not os.path.exists(self.conf_file):
-            print(f'尚未配置: 缺少 .my_conf 文件')
+            print(f'尚未配置: 缺少 .my_conf 文件 {self.conf_file}')
             return
         if not options:
             print(f'传入的参数为空: {options}')

mdbq/config/get_myconf_/345/206/262/347/252/201/346/226/207/344/273/266_Administrator_20240901101505.py

@@ -0,0 +1,131 @@
+# -*- coding: UTF-8 –*-
+import platform
+import getpass
+import socket
+import configparser
+import os
+import sys
+from posixpath import dirname
+from urllib import parse
+from mdbq.config import set_support
+
+
+class MyConf:
+    """
+    读取配置文件信息
+    """
+    def __init__(self, path='support'):
+        self.top_path = os.path.realpath(os.path.dirname(sys.argv[0]))  # 程序运行目录, 打包时使用
+        self.conf_file = os.path.join(self.top_path, path, '.my_conf')
+        self.config = None
+
+    def get_myconf(self, options: list):
+        if not os.path.exists(self.conf_file):
+            print(f'尚未配置: 缺少 .my_conf 文件')
+            return
+        if not options:
+            print(f'传入的参数为空: {options}')
+            return
+        self.config = configparser.ConfigParser()
+        self.config.read(self.conf_file, 'UTF-8')
+        results = []
+        for option in options:
+            try:
+                results.append(self.config.get('database', option))
+            except configparser.NoOptionError:
+                results.append('')
+        return results
+
+
+def select_config_values(target_service, database, path=None):
+    """
+    target_service: 指向: home_lx, aliyun
+    database: 指向: mongodb, mysql
+    """
+    if not path:
+        path = set_support.SetSupport(dirname='support').dirname
+
+    m = MyConf(path=path)
+    options = []
+    if target_service == 'home_lx':  # 1. 家里笔记本
+        if database == 'mongodb':
+            if socket.gethostname() == 'xigua_lx':
+                # 本机自身运行使用 127.0.0.1
+                options = ['username_db_lx_nw', 'password_db_lx_nw', 'host_bd',  'port_db_lx_nw',]
+            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBook-Pro':
+                # 内网地址：正在运行的是 家里笔记本或者台式机，或者 macbook pro
+                options = ['username_db_lx_nw', 'password_db_lx_nw', 'host_db_lx_nw',  'port_db_lx_nw',]
+            else:
+                options = ['username_db_lx', 'password_db_lx', 'host_db_lx', 'port_db_lx']
+
+        elif database == 'mysql':
+            if socket.gethostname() == 'xigua_lx':
+                # 本机自身运行使用 127.0.0.1
+                options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_bd',  'port_mysql_lx_nw',]
+            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBook-Pro':
+                # 内网地址：正在运行的是 家里笔记本或者台式机，或者 macb    ook pro
+                options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_mysql_lx_nw',  'port_mysql_lx_nw',]
+            else:
+                options = ['username_mysql_lx', 'password_mysql_lx', 'host_mysql_lx', 'port_mysql_lx']
+
+    elif target_service == 'home_xigua1':
+        if database == 'mongodb':
+            print('未配置')
+        elif database == 'mysql':
+            if socket.gethostname() == 'xigua_lx':
+                # 本机自身运行使用 127.0.0.1
+                options = ['username_mysql_xigua1_nw', 'password_mysql_xigua1_nw', 'host_mysql_xigua1_nw',  'port_mysql_xigua1_nw',]
+            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'macbook pro':
+                # 内网地址：正在运行的是 家里笔记本或者台式机，或者 macb    ook pro
+                options = ['username_mysql_xigua1_nw', 'password_mysql_xigua1_nw', 'host_bd',  'port_mysql_xigua1_nw',]
+            else:
+                print('未配置')
+                options = ['', '', '', '']
+
+    elif target_service == 'aliyun':  # 2. 阿里云服务器
+        if database == 'mongodb':
+            if socket.gethostname() == 'xigua-cloud':
+                # 阿里云自身运行使用 127.0.0.1
+                options = ['username_db_aliyun', 'password_db_aliyun', 'host_bd', 'port_db_aliyun', ]
+            else:
+                options = ['username_db_aliyun', 'password_db_aliyun', 'host_db_aliyun', 'port_db_aliyun', ]
+        elif database == 'mysql':
+            if socket.gethostname() == 'xigua-cloud':
+                # 阿里云自身运行使用 127.0.0.1
+                options = ['username_mysql_aliyun', 'password_mysql_aliyun', 'host_bd', 'port_mysql_aliyun', ]
+            else:
+                options = ['username_mysql_aliyun', 'password_mysql_aliyun', 'host_mysql_aliyun', 'port_mysql_aliyun', ]
+
+    elif target_service == 'company':  # 3. 公司台式机
+        if database == 'mongodb':
+            options = ['username_db_company_nw', 'password_db_company_nw', 'host_db_company_nw', 'port_db_company_nw', ]
+        elif database == 'mysql':
+            options = ['username_mysql_company_nw', 'password_mysql_company_nw', 'host_mysql_company_nw', 'port_mysql_company_nw', ]
+
+    elif target_service == 'nas':  # 4. 群晖
+        if database == 'mysql':
+            options = ['username_mysql_nas_nw', 'password_mysql_nas_nw', 'host_mysql_nas_nw', 'port_mysql_nas_nw', ]
+
+    value = m.get_myconf(options=options)
+    if not value:
+        return '', '', '', 0
+    if database == 'mongodb':  # mongodb 特殊字符要转码, mysql 不需要转
+        username = parse.quote_plus(str(value[0]).strip())  # 对可能存在的特殊字符进行编码
+        password = parse.quote_plus(str(value[1]).strip())  # 如果密码含有 @、/ 字符，一定要进行编码
+    else:
+        username = str(value[0]).strip()
+        password = str(value[1]).strip()
+    host = str(value[2]).strip()
+    port = int(value[3])
+    return username, password, host, port
+
+
+def main():
+    pass
+
+
+if __name__ == '__main__':
+    # main()
+    r, d, s, g = select_config_values(target_service='home_lx', database='mysql')
+    print(r, d, s, g, type(r), type(d), type(s), type(g))
+    print(f'本机: {platform.system()} // {socket.gethostname()}')

mdbq/mysql/mysql.py

@@ -180,87 +180,87 @@ class MysqlUpload:
             for data in datas:
                 # data 是传进来待处理的数据, 不是数据库数据
                 # data 示例: {'日期': Timestamp('2024-08-27 00:00:00'), '推广费余额': 33299, '品销宝余额': 2930.73, '短信剩余': 67471}
-                # try:
-                cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
-                # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
-                condition = []
-                for k, v in data.items():
-                    condition += [f'`{k}` = "{v}"']
-                condition = ' AND '.join(condition)  # 构建查询条件
-                # print(condition)
-
-                if drop_duplicates:  # 查重插入
-                    sql = "SELECT %s FROM %s WHERE %s" % (cols, table_name, condition)
-                    # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
-                    # print(sql)
-                    cursor.execute(sql)
-                    result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
-                    if not result:  # 数据不存在则插入
-                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
-                        cursor.execute(sql)
-                    # else:
-                    #     print(f'重复数据不插入: {condition[:50]}...')
-                elif icm_update:  # 增量更新, 专门用于聚合数据，其他库不要调用
-                    """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列必须是数据表中唯一主键，值不会发生变化且不会重复，否则可能产生覆盖情况 """
-                    sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, {table_name}))
-                    columns = cursor.fetchall()
-                    cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
-                    update_col = [item for item in cols_exist if item not in icm_update and item != 'id']  # 除了主键外的其他列
-
-                    # unique_keys 示例: `日期`, `推广费余额`
-                    unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
+                try:
+                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
+                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                    values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
                     condition = []
-                    for up_col in icm_update:
-                        condition += [f'`{up_col}` = "{data[up_col]}"']
-                    condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
-                    sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
-                    # print(sql)
-                    # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
-                    cursor.execute(sql)
-                    results = cursor.fetchall()  # results 是数据库取出的数据
-                    if results:  # 有数据返回，再进行增量检查
-                        for result in results:  # results 是数据库数据, data 是传进来的数据
-                            change_col = []  # 发生变化的列名
-                            change_values = []  # 发生变化的数据
-                            for col in update_col:
-                                # 因为 mysql 里面有 decimal 数据类型，要移除末尾的 0 再做比较（df 默认将 5.00 小数截断为 5.0）
-                                df_value = str(data[col])
-                                mysql_value = str(result[col])
-                                if '.' in df_value:
-                                    df_value = re.sub(r'0+$', '', df_value)
-                                    df_value = re.sub(r'\.$', '', df_value)
-                                if '.' in mysql_value:
-                                    mysql_value = re.sub(r'0+$', '', mysql_value)
-                                    mysql_value = re.sub(r'\.$', '', mysql_value)
-                                if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
-                                    # print(f'{data['日期']}{data['商品id']}{col} 列的值有变化，{str(data[col])}  !=  {str(result[col])}')
-                                    change_values += [f"`{col}` = \"{str(data[col])}\""]
-                                    change_col.append(col)
-                            not_change_col = [item for item in update_col if item not in change_col]
-                            # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
-                            if change_values:  # change_values 有数据返回，表示值需要更新
-                                if not_change_col:
-                                    not_change_values = [f'`{col}` = "{str(data[col])}"' for col in not_change_col]
-                                    not_change_values = ' AND '.join(not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
-                                    # print(change_values, not_change_values)
-                                    condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
-                                change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
-                                sql = "UPDATE `%s` SET %s WHERE %s" % (table_name, change_values, condition)
-                                # print(sql)
-                                cursor.execute(sql)
-                    else:  # 没有数据返回，则直接插入数据
-                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                    for k, v in data.items():
+                        condition += [f'`{k}` = "{v}"']
+                    condition = ' AND '.join(condition)  # 构建查询条件
+                    # print(condition)
+
+                    if drop_duplicates:  # 查重插入
+                        sql = "SELECT %s FROM %s WHERE %s" % (cols, table_name, condition)
+                        # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                        # print(sql)
                         cursor.execute(sql)
-                else:
-                    sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
-                    cursor.execute(sql)
-                # except Exception as e:
-                #     # print(data)
-                #     # print(values)
-                #     print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
-                #     # breakpoint()
+                        result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
+                        if not result:  # 数据不存在则插入
+                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
+                            cursor.execute(sql)
+                        # else:
+                        #     print(f'重复数据不插入: {condition[:50]}...')
+                    elif icm_update:  # 增量更新, 专门用于聚合数据，其他库不要调用
+                        """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列必须是数据表中唯一主键，值不会发生变化且不会重复，否则可能产生覆盖情况 """
+                        sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                        cursor.execute(sql, (db_name, {table_name}))
+                        columns = cursor.fetchall()
+                        cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+                        update_col = [item for item in cols_exist if item not in icm_update and item != 'id']  # 除了主键外的其他列
+
+                        # unique_keys 示例: `日期`, `推广费余额`
+                        unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
+                        condition = []
+                        for up_col in icm_update:
+                            condition += [f'`{up_col}` = "{data[up_col]}"']
+                        condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
+                        sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
+                        # print(sql)
+                        # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                        cursor.execute(sql)
+                        results = cursor.fetchall()  # results 是数据库取出的数据
+                        if results:  # 有数据返回，再进行增量检查
+                            for result in results:  # results 是数据库数据, data 是传进来的数据
+                                change_col = []  # 发生变化的列名
+                                change_values = []  # 发生变化的数据
+                                for col in update_col:
+                                    # 因为 mysql 里面有 decimal 数据类型，要移除末尾的 0 再做比较（df 默认将 5.00 小数截断为 5.0）
+                                    df_value = str(data[col])
+                                    mysql_value = str(result[col])
+                                    if '.' in df_value:
+                                        df_value = re.sub(r'0+$', '', df_value)
+                                        df_value = re.sub(r'\.$', '', df_value)
+                                    if '.' in mysql_value:
+                                        mysql_value = re.sub(r'0+$', '', mysql_value)
+                                        mysql_value = re.sub(r'\.$', '', mysql_value)
+                                    if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
+                                        # print(f'{data['日期']}{data['商品id']}{col} 列的值有变化，{str(data[col])}  !=  {str(result[col])}')
+                                        change_values += [f"`{col}` = \"{str(data[col])}\""]
+                                        change_col.append(col)
+                                not_change_col = [item for item in update_col if item not in change_col]
+                                # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
+                                if change_values:  # change_values 有数据返回，表示值需要更新
+                                    if not_change_col:
+                                        not_change_values = [f'`{col}` = "{str(data[col])}"' for col in not_change_col]
+                                        not_change_values = ' AND '.join(not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
+                                        # print(change_values, not_change_values)
+                                        condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
+                                    change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
+                                    sql = "UPDATE `%s` SET %s WHERE %s" % (table_name, change_values, condition)
+                                    # print(sql)
+                                    cursor.execute(sql)
+                        else:  # 没有数据返回，则直接插入数据
+                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                            cursor.execute(sql)
+                    else:
+                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
+                        cursor.execute(sql)
+                except Exception as e:
+                    # print(data)
+                    # print(values)
+                    print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
+                    # breakpoint()
         connection.commit()  # 提交事务
         connection.close()
 

{mdbq-1.5.5.dist-info → mdbq-1.5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.5.5
+Version: 1.5.7
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.5.5.dist-info → mdbq-1.5.7.dist-info}/RECORD

@@ -13,7 +13,8 @@ mdbq/clean/data_clean.py,sha256=BIzc1XCJjJaZyPT6DCRXRCCRwBaeC5_lER0aqYF1P3M,8777
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=WCZ92vCJAy6_ZFeOxWL-U9gArIpyga4xts-s1wKsspY,17268
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/config/get_myconf.py,sha256=bp6bVARZVm3ANj1pmM9hLB8Ao539TUWeM9xxeSsBpzw,5994
+mdbq/config/get_myconf.py,sha256=-CFEW0dQh4OIwVgwK-cL0eVp1LN3PjJgN89d4P5TB9I,6011
+mdbq/config/get_myconf_冲突文件_Administrator_20240901101505.py,sha256=bp6bVARZVm3ANj1pmM9hLB8Ao539TUWeM9xxeSsBpzw,5994
 mdbq/config/products.py,sha256=vIK8DJ-F3XXwvNPK-4OJq2tZITNlL6Sub8QBdoOng8U,5676
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
@@ -24,7 +25,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=o2QRV3JZ_jva8u3ScFiMGcgnInSGFYfADm1-nEKMr4M,43051
+mdbq/mysql/mysql.py,sha256=6hILGosp2fwTp9qgc9IMFTmNzAz0ZG8rznmPxXHY-eA,43331
 mdbq/mysql/s_query.py,sha256=a33aYhW6gAnspIZfQ7l23ePln9-MD1f_ukypr5M0jd8,8018
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -35,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.5.5.dist-info/METADATA,sha256=sojaLXm4UyJOVGaIdmUSed7v7vgOLZWovptcEW_oH5s,245
-mdbq-1.5.5.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.5.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.5.5.dist-info/RECORD,,
+mdbq-1.5.7.dist-info/METADATA,sha256=jKf6D_Ytp6rJxkjmFn_uMhsXKmR6NiUfoe9I_uwILWA,245
+mdbq-1.5.7.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.5.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.5.7.dist-info/RECORD,,