mdbq 2.9.0__py3-none-any.whl → 2.9.3__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -24,7 +24,7 @@ import time
 import re
 import shutil
 import getpass
-
+from sqlalchemy import create_engine
 warnings.filterwarnings('ignore')
 """
 
@@ -1167,7 +1167,24 @@ def upload_dir(path, db_name, collection_name, json_path=None):
                     intersection_keys = dtypes.keys() & old_dt.keys()  # 获取两个字典键的交集
                     dtypes = {k: dtypes[k] for k in intersection_keys}  # 使用交集的键创建新字典
                     df = df.astype(dtypes)  # 再次更新 df 的数据类型
+                df.fillna(0, inplace=True)
+
+                # for col in df.columns.tolist():
+                #     df[col] = df[col].apply(lambda x: 0 if str(x) == '' else x)
+                # print(f'{i}/{count}')
+                # sql_engine = create_engine(
+                #     f"mysql+pymysql://{username}:{password}@{host}:{port}/{db_name}")  # 创建数据库引擎
+                # df.to_sql(
+                #     name=collection_name,
+                #     con=sql_engine,
+                #     if_exists='append',
+                #     index=False,
+                #     chunksize=1000
+                # )
+
+
 
+                #
                 m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
                               move_insert=False,  # 先删除，再插入
                               df_sql = True,
@@ -1201,32 +1218,34 @@ def one_file_to_mysql(file, db_name, table_name):
 
 
 def test():
-    path = os.path.relpath(r'/Users/xigua/Downloads/手淘搜索_本店引流词/2024-05')
+    path = r'C:\同步空间\BaiduSyncdisk\原始文件3\天猫推广报表\品销宝'
 
     results = []
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
-                # print(name)
                 # df = pd.read_excel(os.path.join(root, name), header=0)
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                 # print(name)
                 # if len(df) == 0:
                 #     continue
                 # # df.insert(loc=1, column='店铺名称', value='万里马官方旗舰店')
-                # if '店铺名称' not in df.columns.tolist():
-                #     print(name)
-                #     df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
-                #     df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                df['更新时间'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                # for col in ['更新时间']:
+                #     if col not in df.columns.tolist():
+                #         print(name)
+                #         df[col] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                #         df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
                 # pattern = re.findall(r'\d{4}-\d{2}-\d{2}_\d{4}-\d{2}-\d{2}', name)[0]
                 # new_name = f'py_xg_店铺销售指标_万里马官方旗舰店_{pattern}.csv'
                 # df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
                 # os.remove(os.path.join(root, name))
-                results.append(df)
-    df = pd.concat(results)
-    path = '/Users/xigua/Downloads/手淘搜索_本店引流词'
-    filename = 'py_xg_手淘搜索_本店引流词_万里马官方旗舰店_2024-05_合并.csv'
-    df.to_csv(os.path.join(path, filename), encoding='utf-8_sig', index=False, header=True)
+                # results.append(df)
+    # df = pd.concat(results)
+    # path = '/Users/xigua/Downloads/手淘搜索_本店引流词'
+    # filename = 'py_xg_手淘搜索_本店引流词_万里马官方旗舰店_2024-05_合并.csv'
+    # df.to_csv(os.path.join(path, filename), encoding='utf-8_sig', index=False, header=True)
 
 
 
@@ -1243,15 +1262,18 @@ if __name__ == '__main__':
     #     table_name='超级直播',
     # )
 
-    # # 上传一个目录到指定数据库
-    # db_name = '生意参谋3'
-    # table_name = '手淘搜索_本店引流词'
-    # upload_dir(
-    #     path=os.path.relpath(r'/Users/xigua/Downloads/手淘搜索_本店引流词'),
-    #     db_name=db_name,
-    #     collection_name=table_name,
-    # )
+    # test()
+
+    col = 1
+    if col:
+        # 上传一个目录到指定数据库
+        db_name = '生意参谋3'
+        table_name = '商品排行'
+        upload_dir(
+            path=r'C:\同步空间\BaiduSyncdisk\原始文件3\生意参谋\商品排行',
+            db_name=db_name,
+            collection_name=table_name,
+        )
 
 
-    test()
 

mdbq/aggregation/query_data.py

@@ -204,7 +204,7 @@ class MysqlDatasQuery:
             '场景名字': 1,
             '宝贝id': 1,
             '词类型': 1,
-            '词名字/词包名字': 1,
+            '词名字_词包名字': 1,
             '花费': 1,
             '展现量': 1,
             '点击量': 1,
@@ -246,7 +246,7 @@ class MysqlDatasQuery:
         }
         df = self.download.data_to_df(
             db_name='推广数据2',
-            table_name='超级直播',
+            table_name='超级直播报表_人群',
             start_date=start_date,
             end_date=end_date,
             projection=projection,
@@ -685,7 +685,7 @@ class MysqlDatasQuery:
             }
             df_tm_living = self.download.data_to_df(
                 db_name='推广数据2',
-                table_name='超级直播',
+                table_name='超级直播报表_人群',
                 start_date=start_date,
                 end_date=pd.to_datetime('2024-04-16'),  # 只可以取此日期之前的数据
                 projection=projection,
@@ -866,13 +866,14 @@ class MysqlDatasQuery:
             projection=projection,
         )
         df_dmp.sort_values('日期', ascending=True, ignore_index=True, inplace=True)
-        df_dmp.drop_duplicates(subset=['日期', '人群id', '消耗'], keep='last', inplace=True, ignore_index=True)
+        df_dmp.drop_duplicates(subset=['日期', '人群id', '消耗_元'], keep='last', inplace=True, ignore_index=True)
         df = pd.merge(df_dmp, df_crowd, left_on=['人群id'], right_on=['人群id'], how='left')
         # 清除一些不必要的字符
         df['用户年龄'] = df['用户年龄'].apply(lambda x: '~'.join(re.findall(r'^(\d+).*-(\d+)岁$', str(x))[0]) if '岁' in str(x) else x)
         df['消费能力等级'] = df['消费能力等级'].apply(lambda x: f'L{''.join(re.findall(r'(\d)', str(x)))}' if '购买力' in str(x) else x)
         # df.to_csv('/Users/xigua/Downloads/test3.csv', index=False, header=True, encoding='utf-8_sig')
         # breakpoint()
+        df.rename(columns={'消耗_元': '消耗'}, inplace=True)
         return df
 
 
@@ -999,7 +1000,8 @@ class GroupBy:
             df_pic_lin = df[df['店铺名称'] == '万里马官方旗舰店']
             df_pic = df_pic_lin.groupby(['日期', '商品id'], as_index=False).agg({'花费': 'sum'})
             df_pic = df_pic[~df_pic['商品id'].isin([''])]  # 指定列中删除包含空值的行
-            df_pic = df_pic[(df_pic['日期'] >= f'{year_my}-{last_month.month}-01')]
+            date_obj = datetime.datetime.strptime(f'{year_my}-{last_month.month}-01', '%Y-%m-%d').date()
+            df_pic = df_pic[(df_pic['日期'] >= date_obj)]
             df_pic = df_pic.groupby(['商品id'], as_index=False).agg({'花费': 'sum'})
             df_pic.sort_values('花费', ascending=False, ignore_index=True, inplace=True)
             df_pic.reset_index(inplace=True)
@@ -1166,7 +1168,7 @@ class GroupBy:
                 '直接成交金额': float,
             }, errors='raise')
             if is_maximize:
-                df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '词类型', '词名字/词包名字', '花费', '展现量', '点击量'], as_index=False).agg(
+                df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '词类型', '词名字_词包名字', '花费', '展现量', '点击量'], as_index=False).agg(
                     **{'加购量': ('加购量', np.max),
                        '成交笔数': ('成交笔数', np.max),
                        '成交金额': ('成交金额', np.max),
@@ -1175,7 +1177,7 @@ class GroupBy:
                        }
                 )
             else:
-                df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '词类型', '词名字/词包名字', '花费', '展现量', '点击量'], as_index=False).agg(
+                df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '词类型', '词名字_词包名字', '花费', '展现量', '点击量'], as_index=False).agg(
                     **{
                         '加购量': ('加购量', np.min),
                         '成交笔数': ('成交笔数', np.min),
@@ -1185,7 +1187,7 @@ class GroupBy:
                        }
                 )
             df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
-            df['是否品牌词'] = df['词名字/词包名字'].str.contains('万里马|wanlima', regex=True)
+            df['是否品牌词'] = df['词名字_词包名字'].str.contains('万里马|wanlima', regex=True)
             df['是否品牌词'] = df['是否品牌词'].apply(lambda x: '品牌词' if x else '')
             dir_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\分类配置文件.xlsx'
             dir_file2 = '/Volumes/时尚事业部/01.运营部/0-电商周报-每周五更新/分类配置文件.xlsx'
@@ -1196,17 +1198,17 @@ class GroupBy:
                 # df_fl.rename(columns={'分类1': '词分类'}, inplace=True)
                 df_fl = df_fl[['关键词', '词分类']]
                 # 合并并获取词分类信息
-                df = pd.merge(df, df_fl, left_on=['词名字/词包名字'], right_on=['关键词'], how='left')
+                df = pd.merge(df, df_fl, left_on=['词名字_词包名字'], right_on=['关键词'], how='left')
                 df.pop('关键词')
                 df['词分类'].fillna('', inplace=True)
             if '词分类' in df.columns.tolist():
                 # 这行决定了，从文件中读取的词分类信息优先级高于 ret_keyword 函数的词分类
                 df['词分类'] = df.apply(
-                    lambda x: self.ret_keyword(keyword=str(x['词名字/词包名字']), as_file=False) if x['词分类'] == ''
+                    lambda x: self.ret_keyword(keyword=str(x['词名字_词包名字']), as_file=False) if x['词分类'] == ''
                     else x['词分类'], axis=1
                 )
             else:
-                df['词分类'] = df['词名字/词包名字'].apply(lambda x: self.ret_keyword(keyword=str(x), as_file=False))
+                df['词分类'] = df['词名字_词包名字'].apply(lambda x: self.ret_keyword(keyword=str(x), as_file=False))
             # df.to_csv('/Users/xigua/Downloads/test.csv', index=False, header=True, encoding='utf-8_sig')
             # breakpoint()
             return df
@@ -1546,6 +1548,7 @@ class GroupBy:
             df.drop_duplicates(subset=['场次id'], keep='first', inplace=True, ignore_index=True)
             return df
         elif '多店推广场景_按日聚合' in table_name:
+            df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
             df = df.groupby(
                 ['日期', '店铺名称', '营销场景'],
                 as_index=False).agg(
@@ -2073,8 +2076,49 @@ class GroupBy:
         df.to_excel(os.path.join(path, filename + '.xlsx'), index=index, header=header, engine=engine, freeze_panes=freeze_panes)
 
 
-def g_group():
-    pass
+def date_table():
+    """
+    生成 pbix 使用的日期表
+    """
+    start_date = '2022-01-01'  # 日期表的起始日期
+    yesterday = time.strftime('%Y-%m-%d', time.localtime(time.time() - 86400))
+    dic = pd.date_range(start=start_date, end=yesterday)
+    df = pd.DataFrame(dic, columns=['日期'])
+    df.sort_values('日期', ascending=True, ignore_index=True, inplace=True)
+    df.reset_index(inplace=True)
+    # inplace 添加索引到 df
+    p = df.pop('index')
+    df['月2'] = df['日期']
+    df['月2'] = df['月2'].dt.month
+    df['日期'] = df['日期'].dt.date  # 日期格式保留年月日，去掉时分秒
+    df['年'] = df['日期'].apply(lambda x: str(x).split('-')[0] + '年')
+    df['月'] = df['月2'].apply(lambda x: str(x) + '月')
+    # df.drop('月2', axis=1, inplace=True)
+    mon = df.pop('月2')
+    df['日'] = df['日期'].apply(lambda x: str(x).split('-')[2])
+    df['年月'] = df.apply(lambda x: x['年'] + x['月'], axis=1)
+    df['月日'] = df.apply(lambda x: x['月'] + x['日'] + '日', axis=1)
+    df['第n周'] = df['日期'].apply(lambda x: x.strftime('第%W周'))
+    df['索引'] = p
+    df['月索引'] = mon
+    df.sort_values('日期', ascending=False, ignore_index=True, inplace=True)
+
+    m = mysql.MysqlUpload(
+        username=username,
+        password=password,
+        host=host,
+        port=port,
+    )
+    m.df_to_mysql(
+        df=df,
+        db_name='聚合数据',
+        table_name='日期表',
+        move_insert=True,  # 先删除，再插入
+        df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+        drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+        filename=None,  # 用来追踪处理进度
+        service_database=service_database,  # 用来追踪处理进度
+    )
 
 
 def data_aggregation(months=1, is_juhe=True, less_dict=[]):
@@ -2169,7 +2213,7 @@ def data_aggregation(months=1, is_juhe=True, less_dict=[]):
         {
             '数据库名': '聚合数据',
             '集合名': '天猫_关键词报表',
-            '唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费', '词类型', '词名字/词包名字',],
+            '唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费', '词类型', '词名字_词包名字',],
             '数据主体': sdq.tg_gjc(),
         },
         {
@@ -2311,7 +2355,36 @@ def data_aggregation(months=1, is_juhe=True, less_dict=[]):
 
 
 def main():
-    pass
+    # 更新日期表
+    date_table()
+    # 更新货品年份基准表， 属性设置 3 - 货品年份基准
+    p = products.Products()
+    p.to_mysql()
+
+    system = platform.system()  # 本机系统
+    host_name = socket.gethostname()  # 本机名
+    conf = myconfig.main()
+    db_list = conf[system][host_name]['mysql']['数据库集']
+    db_list = [item for item in db_list if item != '聚合数据']
+    # 清理所有非聚合数据的库
+    optimize_data.op_data(
+        db_name_lists=db_list,
+        days=5,
+        is_mongo=True,
+        is_mysql=True,
+    )
+
+    # 数据聚合
+    query_data.data_aggregation(months=3)
+    time.sleep(60)
+
+    # 清理聚合数据, mongodb 中没有聚合数据，所以只需要清理 mysql 即可
+    optimize_data.op_data(
+        db_name_lists=['聚合数据'],
+        days=100,
+        is_mongo=False,
+        is_mysql=True,
+    )
 
 
 if __name__ == '__main__':

mdbq/mysql/mysql.py

@@ -6,9 +6,12 @@ import re
 import time
 from functools import wraps
 import warnings
+from unittest.mock import inplace
+
 import pymysql
 import numpy as np
 import pandas as pd
+from markdown_it.rules_inline.backticks import regex
 from sqlalchemy import create_engine
 import os
 import calendar
@@ -41,6 +44,67 @@ warnings.filterwarnings('ignore')
 """
 
 
+def is_valid_date(date_string):
+    """
+    判断是否是日期格式, 且允许没有前导零, 且允许带时间
+    纯日期格式： 返回 1
+    日期+时间： 返回 2
+    """
+    date_pattern = r"^(\d{4})-(0?[1-9]|1[0-2])-(0?[1-9]|[12]\d|3[01])$"
+    match = re.match(date_pattern, str(date_string))  # 判断纯日期格式：2024-11-09
+    if match is None:
+        date_pattern = r".*\d+:\d+:\d+$"
+        match = re.match(date_pattern, date_string)  # 判断日期+时间：2024-11-09 00:36:45
+        if match is not None:
+            return 2
+    else:
+        return 1
+
+
+def is_integer(int_str):
+    """ 判断是否整数, 允许包含千分位分隔符, 允许科学计数法 """
+    # 如果是科学计数法
+    match = re.findall(r'^[-+]?(\d+)\.(\d+)[eE][-+]?(\d+)$', str(int_str))
+    if match:
+        if len(match[0]) == 3:
+            if int(match[0][0]) == 0:  # 0 开头
+                if int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
+                    return False
+            else:  # 不是 0 开头
+                if len(match[0][0]) + int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
+                    return False
+            if int(match[0][2]) >= len(match[0][1]):
+                return True
+            else:
+                return False
+    # 如果是普通数字, 且允许千分符
+    __pattern = r'^[-+]?\d{1,3}(,\d{3}){0,3}$|^[-+]?\d{1,9}$'
+    return re.match(__pattern, str(int_str)) is not None
+
+
+def count_decimal_places(num_str):
+    """ 计算小数位数, 允许科学计数法 """
+    match = re.match(r'^[-+]?\d+(\.\d+)?([eE][-+]?\d+)?$', str(num_str))
+    if match:
+        # 如果是科学计数法
+        match = re.findall(r'(\d+)\.(\d+)[eE][-+]?(\d+)$', str(num_str))
+        if match:
+            if len(match[0]) == 3:
+                if int(match[0][2]) < len(match[0][1]):
+                    # count_int 清除整数部分开头的 0 并计算整数位数
+                    count_int = len(re.sub('^0+', '', str(match[0][0]))) + int(match[0][2])
+                    # 计算小数位数
+                    count_float = len(match[0][1]) - int(match[0][2])
+                    return count_int, count_float
+        # 如果是普通小数
+        match = re.findall(r'(\d+)\.(\d+)$', str(num_str))
+        if match:
+            count_int = len(re.sub('^0+', '', str(match[0][0])))
+            count_float = len(match[0][1])
+            return count_int, count_float  # 计算小数位数
+    return 0, 0
+
+
 class MysqlUpload:
     def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
         self.username = username
@@ -68,6 +132,229 @@ class MysqlUpload:
 
         return wrapper
 
+    def dict_to_mysql(self, db_name, table_name, dict_data, icm_update=None, main_key=None, unique_main_key=None, index_length=100, set_type=None):
+        """
+        插入字典数据
+        dict_data： 字典
+        main_key： 指定索引列, 通常用日期列，默认会设置日期为索引
+        unique_main_key： 指定唯一索引列
+        index_length: 索引长度
+        icm_update: 增量更正，指定后 main_key 只用于检查/创建列，不能更新数据
+        set_type: {}
+        """
+        if not main_key:
+            main_key = []
+        if not unique_main_key:
+            unique_main_key = []
+        connection = pymysql.connect(**self.config)  # 连接数据库
+        with connection.cursor() as cursor:
+            cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
+            database_exists = cursor.fetchone()
+            if not database_exists:
+                # 如果数据库不存在，则新建
+                if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
+                    self.config.update({'charset': 'utf8mb4_unicode_ci'})
+                if '192.168.1.100' in str(self.host):
+                    sql = f"CREATE DATABASE `{db_name}`"
+                else:
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+                cursor.execute(sql)
+                connection.commit()
+                print(f"创建Database: {db_name}")
+
+        self.config.update({'database': db_name})  # 添加更新 config 字段
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        with connection.cursor() as cursor:
+            # 1. 查询表, 不存在则创建一个空表
+            sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
+            cursor.execute(sql, (table_name))
+            if not cursor.fetchone():
+                sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (id INT AUTO_INCREMENT PRIMARY KEY);"
+                cursor.execute(sql)
+                print(f'创建 mysql 表: {table_name}')
+
+            # 根据 dict_data 的值添加指定的数据类型
+            dtypes, dict_data = self.cover_dict_dtypes(dict_data=dict_data)  # {'店铺名称': 'mediumtext',...}
+            if set_type:
+                dtypes.update(set_type)  # 自定义的数据类型
+
+            # 检查列
+            sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s;"
+            cursor.execute(sql, (db_name, table_name))
+            col_exist = [item['COLUMN_NAME'] for item in cursor.fetchall()]  # 已存在的所有列
+            col_not_exist = [col for col in dict_data.keys() if col not in col_exist]  # 不存在的列
+            # 不存在则新建列
+            if col_not_exist:  # 数据表中不存在的列
+                for col in col_not_exist:
+                    #  创建列，需转义
+                    sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
+                    cursor.execute(sql)
+                    print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
+
+                    if col in main_key or col == '日期':
+                        sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
+                        print(f"设置为索引: {col}({dtypes[col]})")
+                        cursor.execute(sql)
+                    if col in unique_main_key:
+                        if dtypes[col] == 'mediumtext':
+                            sql = f"ALTER TABLE {table_name} ADD UNIQUE (`{col}`({index_length}))"
+                        else:
+                            sql = f"ALTER TABLE {table_name} ADD UNIQUE (`{col}`)"
+                        cursor.execute(sql)
+                    # if col in main_key or col in unique_main_key:
+                    #     sql = f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = %s"
+                    #     cursor.execute(sql, (col))
+                    #     result = cursor.fetchone()  # 检查索引是否存在
+                    #     if not result:
+                    #         if col in main_key:
+                    #             sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
+                    #             print(f"设置为索引: {col}({dtypes[col]})")
+                    #             cursor.execute(sql)
+                    #         elif col in unique_main_key:
+                    #             if dtypes[col] == 'mediumtext':
+                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`({index_length}));"
+                    #             else:
+                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`);"
+                    #             print(f"设置唯一索引: {col}({dtypes[col]})")
+                    #             print(sql)
+                    #             cursor.execute(sql)
+            connection.commit()  # 提交事务
+            """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+            """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+            # 处理插入的数据
+            if icm_update:
+                """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列组合是数据表中唯一，值不会发生变化且不会重复，否则可能产生覆盖 """
+                sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                cursor.execute(sql, (db_name, {table_name}))
+                columns = cursor.fetchall()
+                cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+                update_col = [item for item in cols_exist if item not in icm_update and item != 'id']  # 除了主键外的其他列
+
+                # unique_keys 示例: `日期`, `余额`
+                unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
+                condition = []
+                for up_col in icm_update:
+                    condition += [f'`{up_col}` = "{dict_data[up_col]}"']
+                condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
+                sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
+                # print(sql)
+                # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                cursor.execute(sql)
+                results = cursor.fetchall()  # results 是数据库取出的数据
+                if results:  # 有数据返回，再进行增量检查
+                    for result in results:  # results 是数据库数据, dict_data 是传进来的数据
+                        change_col = []  # 发生变化的列名
+                        change_values = []  # 发生变化的数据
+                        for col in update_col:
+                            # 因为 mysql 里面有 decimal 数据类型，要移除末尾的 0 再做比较（df 默认将 5.00 小数截断为 5.0）
+                            df_value = str(dict_data[col])
+                            mysql_value = str(result[col])
+                            if '.' in df_value:
+                                df_value = re.sub(r'0+$', '', df_value)
+                                df_value = re.sub(r'\.$', '', df_value)
+                            if '.' in mysql_value:
+                                mysql_value = re.sub(r'0+$', '', mysql_value)
+                                mysql_value = re.sub(r'\.$', '', mysql_value)
+                            if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
+                                # print(f'{dict_data['日期']}{dict_data['商品id']}{col} 列的值有变化，{str(dict_data[col])}  !=  {str(result[col])}')
+                                change_values += [f"`{col}` = \"{str(dict_data[col])}\""]
+                                change_col.append(col)
+                        not_change_col = [item for item in update_col if item not in change_col]
+                        # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
+                        if change_values:  # change_values 有数据返回，表示值需要更新
+                            if not_change_col:
+                                not_change_values = [f'`{col}` = "{str(dict_data[col])}"' for col in not_change_col]
+                                not_change_values = ' AND '.join(
+                                    not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
+                                # print(change_values, not_change_values)
+                                condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
+                            change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
+                            sql = "UPDATE `%s` SET %s WHERE %s" % (table_name, change_values, condition)
+                            # print(sql)
+                            cursor.execute(sql)
+                else:  # 没有数据返回，则直接插入数据
+                    cols = ', '.join(f"`{item}`" for item in dict_data.keys())  # 列名需要转义
+                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                    values = ', '.join([f'"{item}"' for item in dict_data.values()])  # 值要加引号
+                    sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                    cursor.execute(sql)
+                connection.commit()  # 提交数据库
+                connection.close()
+                return
+
+            # 构建 keys
+            keys_data = ', '.join([f'`{str(item)}`' for item in dict_data.keys()])
+            # 构建 values
+            values_data = ', '.join(f'"{str(item)}"' for item in dict_data.values())
+            # 构建其他键值，重复时要更新的其他键
+            if main_key:
+                for col in main_key:
+                    del dict_data[col]
+            if unique_main_key:
+                for col in unique_main_key:
+                    del dict_data[col]
+            update_datas = ', '.join([f'{k} = VALUES({k})' for k, v in dict_data.items()])
+
+            # 构建 sql
+            sql = f"INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s" % (table_name, keys_data, values_data, update_datas)
+            cursor.execute(sql)
+            connection.commit()  # 提交数据库
+        connection.close()
+
+    def cover_dict_dtypes(self, dict_data):
+        if not dict_data:
+            print(f'mysql.py -> MysqlUpload -> cover_dict_dtypes -> 传入的字典不能为空')
+            return
+        __res_dict = {}
+        new_dict_data = {}
+        for k, v in dict_data.items():
+            k = str(k).lower()
+            k = re.sub(r'[()\-，,$&~^、 （）\"\'“”=·/。》《><！!`]', '_', k, re.IGNORECASE)
+            k = k.replace('）', '')
+            k = re.sub(r'_{2,}', '_', k)
+            k = re.sub(r'_+$', '', k)
+            if str(v) == '':
+                v = 0
+            v = str(v)
+            v = re.sub('^-$|^--$|^nan$|^null$', '0', v, re.I)
+            v = re.sub(',|="|"', '', v, re.I)
+            if re.findall(r'^[-+]?\d+\.?\d*%$', v):
+                v = str(float(v.rstrip("%")) / 100)
+
+            result1 = re.findall(r'编码|_?id|货号|款号|文件大小', k, re.IGNORECASE)
+            result2 = re.findall(r'占比$|投产$|产出$|同比$|环比$|roi$|率$', k, re.IGNORECASE)
+            date_type = is_valid_date(v)  # 判断日期时间
+            int_num = is_integer(v)  # 判断整数
+            count_int, count_float = count_decimal_places(v)  # 判断小数，返回小数位数
+            if result1:  # 京东sku/spu商品信息
+                __res_dict.update({k: 'mediumtext'})
+            elif str(v) == '':
+                __res_dict.update({k: 'mediumtext'})
+            elif result2:  # 小数
+                __res_dict.update({k: 'decimal(10,4)'})
+            elif date_type == 1:  # 纯日期
+                __res_dict.update({k: 'DATE'})
+            elif date_type == 2:  # 日期+时间
+                __res_dict.update({k: 'DATETIME'})
+            elif int_num:
+                __res_dict.update({k: 'INT'})
+            elif count_float > 0:
+                if count_int + count_float > 10:
+                    if count_float > 5:
+                        v = round(float(v), 4)
+                    __res_dict.update({k: 'decimal(12,4)'})
+                elif count_float >= 6:
+                    __res_dict.update({k: 'decimal(12,6)'})
+                elif count_float >= 4:
+                    __res_dict.update({k: 'decimal(10,4)'})
+                else:
+                    __res_dict.update({k: 'decimal(10,2)'})
+            else:
+                __res_dict.update({k: 'mediumtext'})
+            new_dict_data.update({k: v})
+        return __res_dict, new_dict_data
+
     @try_except
     def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'xigua_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, reset_id=False):
         """
@@ -247,9 +534,7 @@ class MysqlUpload:
                 # data 是传进来待处理的数据, 不是数据库数据
                 # data 示例: {'日期': Timestamp('2024-08-27 00:00:00'), '推广费余额': 33299, '品销宝余额': 2930.73, '短信剩余': 67471}
                 try:
-                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
-                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                    values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
+
                     condition = []
                     for k, v in data.items():
                         condition += [f'`{k}` = "{v}"']
@@ -877,3 +1162,7 @@ if __name__ == '__main__':
     data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = data['username'], data['password'], data['host'], data['port']
     print(username, password, host, port)
+
+    ss = '2024-11-08'
+    ss= re.sub(r'\\N', '0', ss)
+    print(ss, '111')

{mdbq-2.9.0.dist-info → mdbq-2.9.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.9.0
+Version: 2.9.3
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.9.0.dist-info → mdbq-2.9.3.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=VtGP7KhUY-NUBA2CXt50hYuGOZ1bU_NeaBBvAUxYgTg,72338
+mdbq/aggregation/aggregation.py,sha256=IVh9SFO1yp12qDBuEOWTi9SAytYktKBrsPJNPuDetSM,73254
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=79uwiM2WqNNFxGpE2wKz742PRq-ZGgFjdOV0vgptHdY,3513
-mdbq/aggregation/query_data.py,sha256=iRgPljgOPE7dzhaaVxRXOEOOKQTmWg6sGsDplNLTvQw,100177
+mdbq/aggregation/query_data.py,sha256=zut8WyyAKTULfGWMltyQYqsVsIaBDUU8E3w2_UL4hbA,103248
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -28,7 +28,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=apcj0WDdbrHr7UzO2kjcesDxDUlWxG4KcIpI1mBuwMk,46152
+mdbq/mysql/mysql.py,sha256=tKkgjbOvy5uIn7Z-ws_biS-04-UHnr5rKqNvtWr_Yss,62024
 mdbq/mysql/recheck_mysql.py,sha256=jHQSlQy0PlQ_EYICQv_2nairUX3t6OIwPtSELKIpjkY,8702
 mdbq/mysql/s_query.py,sha256=bgNNIqYLDCHjD5KTFcm6x4u74selpAGs5ouJYuqX86k,8447
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -45,7 +45,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=jHrdGWBJQaSywx7V-U4YuM6vWkwC5SR5tTOOdB3YU_c,17306
-mdbq-2.9.0.dist-info/METADATA,sha256=s0ZT23hnWNpOZ5YBPTKigR8XftMlBOTCzH_7Lfuamig,243
-mdbq-2.9.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.9.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.9.0.dist-info/RECORD,,
+mdbq-2.9.3.dist-info/METADATA,sha256=fL1JR-lJNlMr2cIzQIEO460TetP9yzBerJPCJYnBRQ8,243
+mdbq-2.9.3.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-2.9.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.9.3.dist-info/RECORD,,

{mdbq-2.9.0.dist-info → mdbq-2.9.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any