mdbq 2.9.2__py3-none-any.whl → 2.9.4__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -24,7 +24,7 @@ import time
 import re
 import shutil
 import getpass
-
+from sqlalchemy import create_engine
 warnings.filterwarnings('ignore')
 """
 
@@ -1167,7 +1167,24 @@ def upload_dir(path, db_name, collection_name, json_path=None):
                     intersection_keys = dtypes.keys() & old_dt.keys()  # 获取两个字典键的交集
                     dtypes = {k: dtypes[k] for k in intersection_keys}  # 使用交集的键创建新字典
                     df = df.astype(dtypes)  # 再次更新 df 的数据类型
+                df.fillna(0, inplace=True)
+
+                # for col in df.columns.tolist():
+                #     df[col] = df[col].apply(lambda x: 0 if str(x) == '' else x)
+                # print(f'{i}/{count}')
+                # sql_engine = create_engine(
+                #     f"mysql+pymysql://{username}:{password}@{host}:{port}/{db_name}")  # 创建数据库引擎
+                # df.to_sql(
+                #     name=collection_name,
+                #     con=sql_engine,
+                #     if_exists='append',
+                #     index=False,
+                #     chunksize=1000
+                # )
+
+
 
+                #
                 m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
                               move_insert=False,  # 先删除，再插入
                               df_sql = True,
@@ -1201,33 +1218,50 @@ def one_file_to_mysql(file, db_name, table_name):
 
 
 def test():
-    path = os.path.relpath(r'/Users/xigua/Downloads/手淘搜索_本店引流词/2024-05')
+    path = r'/Users/xigua/Downloads/DMP报表'
 
     results = []
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
-                # print(name)
                 # df = pd.read_excel(os.path.join(root, name), header=0)
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                results.append(df)
                 # print(name)
-                # if len(df) == 0:
-                #     continue
-                # # df.insert(loc=1, column='店铺名称', value='万里马官方旗舰店')
-                # if '店铺名称' not in df.columns.tolist():
-                #     print(name)
-                #     df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
-                #     df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                if len(df) == 0:
+                    continue
+                if '达摩盘消耗占比' in df.columns.tolist():
+                    print(name)
+                    df.pop('达摩盘消耗占比')
+                    # df.insert(loc=1, column='店铺名称', value='万里马官方旗舰店')
+                    # df['更新时间'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                    df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                # for col in ['更新时间']:
+                #     if col not in df.columns.tolist():
+                #         print(name)
+                #         df[col] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                #         df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
                 # pattern = re.findall(r'\d{4}-\d{2}-\d{2}_\d{4}-\d{2}-\d{2}', name)[0]
                 # new_name = f'py_xg_店铺销售指标_万里马官方旗舰店_{pattern}.csv'
                 # df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
                 # os.remove(os.path.join(root, name))
-                results.append(df)
-    df = pd.concat(results)
-    path = '/Users/xigua/Downloads/手淘搜索_本店引流词'
-    filename = 'py_xg_手淘搜索_本店引流词_万里马官方旗舰店_2024-05_合并.csv'
-    df.to_csv(os.path.join(path, filename), encoding='utf-8_sig', index=False, header=True)
-
+                # results.append(df)
+    # df = pd.concat(results)
+    # df.drop_duplicates(
+    #     subset=[
+    #         '日期',
+    #         '店铺名称',
+    #         '报表类型',
+    #         '消耗',
+    #         '展现量',
+    #         '点击量',
+    #     ], keep='last', inplace=True, ignore_index=True)
+    # df.fillna(0, inplace=True)
+    # for col in df.columns.tolist():
+    #     df[col] = df[col].apply(lambda x: 0 if str(x) == '' else x)
+    # path = '/Users/xigua/Downloads'
+    # filename = '品销宝_2024年_合并.csv'
+    # df.to_csv(os.path.join(path, filename), encoding='utf-8_sig', index=False, header=True)
 
 
 if __name__ == '__main__':
@@ -1238,20 +1272,23 @@ if __name__ == '__main__':
     #
     # # 上传 1 个文件到数据库
     # one_file_to_mysql(
-    #     file=r'/Users/xi',
-    #     db_name='推广数据2',
-    #     table_name='超级直播',
+    #     file=r'/Users/xigua/Downloads/DMP报表_2024-10-23_2024-10-29.csv',
+    #     db_name='达摩盘3',
+    #     table_name='dmp人群报表',
     # )
 
-    # # 上传一个目录到指定数据库
-    # db_name = '生意参谋3'
-    # table_name = '手淘搜索_本店引流词'
-    # upload_dir(
-    #     path=os.path.relpath(r'/Users/xigua/Downloads/手淘搜索_本店引流词'),
-    #     db_name=db_name,
-    #     collection_name=table_name,
-    # )
+    # test()
+
+    col = 1
+    if col:
+        # 上传一个目录到指定数据库
+        db_name = '达摩盘3'
+        table_name = 'dmp人群报表'
+        upload_dir(
+            path=r'/Users/xigua/Downloads/DMP报表',
+            db_name=db_name,
+            collection_name=table_name,
+        )
 
 
-    test()
 

mdbq/aggregation/query_data.py

@@ -6,6 +6,7 @@ from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.aggregation import optimize_data
 from mdbq.config import myconfig
+from mdbq.config import products
 import datetime
 from dateutil.relativedelta import relativedelta
 import pandas as pd
@@ -15,6 +16,7 @@ import platform
 import getpass
 import json
 import os
+import time
 
 """
 程序用于下载数据库(调用 s_query.py 下载并清洗), 并对数据进行聚合清洗, 不会更新数据库信息;
@@ -204,7 +206,7 @@ class MysqlDatasQuery:
             '场景名字': 1,
             '宝贝id': 1,
             '词类型': 1,
-            '词名字/词包名字': 1,
+            '词名字_词包名字': 1,
             '花费': 1,
             '展现量': 1,
             '点击量': 1,
@@ -246,7 +248,7 @@ class MysqlDatasQuery:
         }
         df = self.download.data_to_df(
             db_name='推广数据2',
-            table_name='超级直播',
+            table_name='超级直播报表_人群',
             start_date=start_date,
             end_date=end_date,
             projection=projection,
@@ -685,7 +687,7 @@ class MysqlDatasQuery:
             }
             df_tm_living = self.download.data_to_df(
                 db_name='推广数据2',
-                table_name='超级直播',
+                table_name='超级直播报表_人群',
                 start_date=start_date,
                 end_date=pd.to_datetime('2024-04-16'),  # 只可以取此日期之前的数据
                 projection=projection,
@@ -855,8 +857,7 @@ class MysqlDatasQuery:
         df_crowd.sort_values('日期', ascending=True, ignore_index=True, inplace=True)
         df_crowd.drop_duplicates(subset=['人群id',], keep='last', inplace=True, ignore_index=True)
         df_crowd.pop('日期')
-        # print(df_crowd)
-
+        df_crowd = df_crowd.astype({'人群id': 'int64'}, errors='ignore')
         projection = {}
         df_dmp = self.download.data_to_df(
             db_name='达摩盘3',
@@ -865,14 +866,16 @@ class MysqlDatasQuery:
             end_date=end_date,
             projection=projection,
         )
+        df_dmp = df_dmp.astype({'人群id': 'int64'}, errors='ignore')
         df_dmp.sort_values('日期', ascending=True, ignore_index=True, inplace=True)
-        df_dmp.drop_duplicates(subset=['日期', '人群id', '消耗'], keep='last', inplace=True, ignore_index=True)
+        df_dmp.drop_duplicates(subset=['日期', '人群id', '消耗_元'], keep='last', inplace=True, ignore_index=True)
         df = pd.merge(df_dmp, df_crowd, left_on=['人群id'], right_on=['人群id'], how='left')
         # 清除一些不必要的字符
         df['用户年龄'] = df['用户年龄'].apply(lambda x: '~'.join(re.findall(r'^(\d+).*-(\d+)岁$', str(x))[0]) if '岁' in str(x) else x)
         df['消费能力等级'] = df['消费能力等级'].apply(lambda x: f'L{''.join(re.findall(r'(\d)', str(x)))}' if '购买力' in str(x) else x)
         # df.to_csv('/Users/xigua/Downloads/test3.csv', index=False, header=True, encoding='utf-8_sig')
         # breakpoint()
+        df.rename(columns={'消耗_元': '消耗'}, inplace=True)
         return df
 
 
@@ -999,7 +1002,8 @@ class GroupBy:
             df_pic_lin = df[df['店铺名称'] == '万里马官方旗舰店']
             df_pic = df_pic_lin.groupby(['日期', '商品id'], as_index=False).agg({'花费': 'sum'})
             df_pic = df_pic[~df_pic['商品id'].isin([''])]  # 指定列中删除包含空值的行
-            df_pic = df_pic[(df_pic['日期'] >= f'{year_my}-{last_month.month}-01')]
+            date_obj = datetime.datetime.strptime(f'{year_my}-{last_month.month}-01', '%Y-%m-%d').date()
+            df_pic = df_pic[(df_pic['日期'] >= date_obj)]
             df_pic = df_pic.groupby(['商品id'], as_index=False).agg({'花费': 'sum'})
             df_pic.sort_values('花费', ascending=False, ignore_index=True, inplace=True)
             df_pic.reset_index(inplace=True)
@@ -1166,7 +1170,7 @@ class GroupBy:
                 '直接成交金额': float,
             }, errors='raise')
             if is_maximize:
-                df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '词类型', '词名字/词包名字', '花费', '展现量', '点击量'], as_index=False).agg(
+                df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '词类型', '词名字_词包名字', '花费', '展现量', '点击量'], as_index=False).agg(
                     **{'加购量': ('加购量', np.max),
                        '成交笔数': ('成交笔数', np.max),
                        '成交金额': ('成交金额', np.max),
@@ -1175,7 +1179,7 @@ class GroupBy:
                        }
                 )
             else:
-                df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '词类型', '词名字/词包名字', '花费', '展现量', '点击量'], as_index=False).agg(
+                df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '词类型', '词名字_词包名字', '花费', '展现量', '点击量'], as_index=False).agg(
                     **{
                         '加购量': ('加购量', np.min),
                         '成交笔数': ('成交笔数', np.min),
@@ -1185,7 +1189,7 @@ class GroupBy:
                        }
                 )
             df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
-            df['是否品牌词'] = df['词名字/词包名字'].str.contains('万里马|wanlima', regex=True)
+            df['是否品牌词'] = df['词名字_词包名字'].str.contains('万里马|wanlima', regex=True)
             df['是否品牌词'] = df['是否品牌词'].apply(lambda x: '品牌词' if x else '')
             dir_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\分类配置文件.xlsx'
             dir_file2 = '/Volumes/时尚事业部/01.运营部/0-电商周报-每周五更新/分类配置文件.xlsx'
@@ -1196,17 +1200,17 @@ class GroupBy:
                 # df_fl.rename(columns={'分类1': '词分类'}, inplace=True)
                 df_fl = df_fl[['关键词', '词分类']]
                 # 合并并获取词分类信息
-                df = pd.merge(df, df_fl, left_on=['词名字/词包名字'], right_on=['关键词'], how='left')
+                df = pd.merge(df, df_fl, left_on=['词名字_词包名字'], right_on=['关键词'], how='left')
                 df.pop('关键词')
                 df['词分类'].fillna('', inplace=True)
             if '词分类' in df.columns.tolist():
                 # 这行决定了，从文件中读取的词分类信息优先级高于 ret_keyword 函数的词分类
                 df['词分类'] = df.apply(
-                    lambda x: self.ret_keyword(keyword=str(x['词名字/词包名字']), as_file=False) if x['词分类'] == ''
+                    lambda x: self.ret_keyword(keyword=str(x['词名字_词包名字']), as_file=False) if x['词分类'] == ''
                     else x['词分类'], axis=1
                 )
             else:
-                df['词分类'] = df['词名字/词包名字'].apply(lambda x: self.ret_keyword(keyword=str(x), as_file=False))
+                df['词分类'] = df['词名字_词包名字'].apply(lambda x: self.ret_keyword(keyword=str(x), as_file=False))
             # df.to_csv('/Users/xigua/Downloads/test.csv', index=False, header=True, encoding='utf-8_sig')
             # breakpoint()
             return df
@@ -1546,6 +1550,7 @@ class GroupBy:
             df.drop_duplicates(subset=['场次id'], keep='first', inplace=True, ignore_index=True)
             return df
         elif '多店推广场景_按日聚合' in table_name:
+            df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
             df = df.groupby(
                 ['日期', '店铺名称', '营销场景'],
                 as_index=False).agg(
@@ -1956,7 +1961,7 @@ class GroupBy:
         )
         return df
 
-    @try_except
+    # @try_except
     def performance_jd(self, jd_tg=True):
         jdtg, sku_sales = self.data_jdtg['京东_京准通'], self.data_jdtg['京东_sku_商品明细']
         jdtg = jdtg.groupby(['日期', '跟单sku_id'],
@@ -1970,6 +1975,7 @@ class GroupBy:
         df = df[['日期', '商品id', '货号', '成交单量', '成交金额', '成本价']]
         df['商品id'] = df['商品id'].astype(str)
         jdtg['跟单sku_id'] = jdtg['跟单sku_id'].astype(str)
+        jdtg = jdtg.astype({'日期': 'datetime64[ns]'}, errors='raise')
         if jd_tg is True:
             # 完整的数据表，包含全店所有推广、销售数据
             df = pd.merge(df, jdtg, how='left', left_on=['日期', '商品id'], right_on=['日期', '跟单sku_id'])  # df 合并推广表
@@ -2073,8 +2079,49 @@ class GroupBy:
         df.to_excel(os.path.join(path, filename + '.xlsx'), index=index, header=header, engine=engine, freeze_panes=freeze_panes)
 
 
-def g_group():
-    pass
+def date_table():
+    """
+    生成 pbix 使用的日期表
+    """
+    start_date = '2022-01-01'  # 日期表的起始日期
+    yesterday = time.strftime('%Y-%m-%d', time.localtime(time.time() - 86400))
+    dic = pd.date_range(start=start_date, end=yesterday)
+    df = pd.DataFrame(dic, columns=['日期'])
+    df.sort_values('日期', ascending=True, ignore_index=True, inplace=True)
+    df.reset_index(inplace=True)
+    # inplace 添加索引到 df
+    p = df.pop('index')
+    df['月2'] = df['日期']
+    df['月2'] = df['月2'].dt.month
+    df['日期'] = df['日期'].dt.date  # 日期格式保留年月日，去掉时分秒
+    df['年'] = df['日期'].apply(lambda x: str(x).split('-')[0] + '年')
+    df['月'] = df['月2'].apply(lambda x: str(x) + '月')
+    # df.drop('月2', axis=1, inplace=True)
+    mon = df.pop('月2')
+    df['日'] = df['日期'].apply(lambda x: str(x).split('-')[2])
+    df['年月'] = df.apply(lambda x: x['年'] + x['月'], axis=1)
+    df['月日'] = df.apply(lambda x: x['月'] + x['日'] + '日', axis=1)
+    df['第n周'] = df['日期'].apply(lambda x: x.strftime('第%W周'))
+    df['索引'] = p
+    df['月索引'] = mon
+    df.sort_values('日期', ascending=False, ignore_index=True, inplace=True)
+
+    m = mysql.MysqlUpload(
+        username=username,
+        password=password,
+        host=host,
+        port=port,
+    )
+    m.df_to_mysql(
+        df=df,
+        db_name='聚合数据',
+        table_name='日期表',
+        move_insert=True,  # 先删除，再插入
+        df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+        drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+        filename=None,  # 用来追踪处理进度
+        service_database=service_database,  # 用来追踪处理进度
+    )
 
 
 def data_aggregation(months=1, is_juhe=True, less_dict=[]):
@@ -2169,7 +2216,7 @@ def data_aggregation(months=1, is_juhe=True, less_dict=[]):
         {
             '数据库名': '聚合数据',
             '集合名': '天猫_关键词报表',
-            '唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费', '词类型', '词名字/词包名字',],
+            '唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费', '词类型', '词名字_词包名字',],
             '数据主体': sdq.tg_gjc(),
         },
         {
@@ -2310,13 +2357,40 @@ def data_aggregation(months=1, is_juhe=True, less_dict=[]):
         )
 
 
-def main():
-    pass
+def main(days=100, months=3):
+    # 更新日期表
+    date_table()
+    # 更新货品年份基准表， 属性设置 3 - 货品年份基准
+    p = products.Products()
+    p.to_mysql()
 
-
-if __name__ == '__main__':
+    # 数据聚合
     data_aggregation(
-        months=3,
+        months=months,
         is_juhe=True,  # 生成聚合表
         # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
     )
+    time.sleep(60)
+
+    system = platform.system()  # 本机系统
+    host_name = socket.gethostname()  # 本机名
+    conf = myconfig.main()
+    db_list = conf[system][host_name]['mysql']['数据库集']
+    # 清理所有库
+    optimize_data.op_data(
+        db_name_lists=db_list,
+        days=days,
+        is_mongo=False,
+        is_mysql=True,
+    )
+
+
+if __name__ == '__main__':
+    main(days=100, months=3)
+
+    # data_aggregation(
+    #     months=3,
+    #     is_juhe=True,  # 生成聚合表
+    #     # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
+    # )
+

mdbq/config/products.py

@@ -124,25 +124,39 @@ class Products:
         ]
         self.datas += my_datas
 
+
     def to_mysql(self):
         self.update_my_datas()
         df = pd.DataFrame(self.datas)
-        m = mysql.MysqlUpload(
+        m_engine = mysql.MysqlUpload(
             username=username,
             password=password,
             host=host,
             port=port,
         )
-        m.df_to_mysql(
-            df=df,
-            db_name='属性设置3',
-            table_name='货品年份基准',
-            move_insert = False,
-            df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
-            drop_duplicates=True,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
-            icm_update=[],
-            service_database=service_database,  # 用来追踪处理进度
-        )
+        for dict_data in df.to_dict(orient='records'):
+            m_engine.dict_to_mysql(
+                db_name='属性设置3',
+                table_name='货品年份基准',
+                dict_data=dict_data,
+                # icm_update=['日期', '店铺名称', '宝贝id'],  # 唯一组合键
+                unique_main_key=['商品id'],
+                set_type={
+                    '商品id': 'mediumtext',
+                    '平台': 'mediumtext',
+                    '上市年份': 'mediumtext',
+                },
+            )
+        # m.df_to_mysql(
+        #     df=df,
+        #     db_name='属性设置3',
+        #     table_name='货品年份基准',
+        #     move_insert = False,
+        #     df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+        #     drop_duplicates=True,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+        #     icm_update=[],
+        #     service_database=service_database,  # 用来追踪处理进度
+        # )
 
     def market_date(self, product_id: int):
         try:

mdbq/mysql/mysql.py

@@ -6,9 +6,12 @@ import re
 import time
 from functools import wraps
 import warnings
+from unittest.mock import inplace
+
 import pymysql
 import numpy as np
 import pandas as pd
+from markdown_it.rules_inline.backticks import regex
 from sqlalchemy import create_engine
 import os
 import calendar
@@ -42,7 +45,11 @@ warnings.filterwarnings('ignore')
 
 
 def is_valid_date(date_string):
-    """ 判断是否是日期格式, 且允许没有前导零, 且允许带时间 """
+    """
+    判断是否是日期格式, 且允许没有前导零, 且允许带时间
+    纯日期格式： 返回 1
+    日期+时间： 返回 2
+    """
     date_pattern = r"^(\d{4})-(0?[1-9]|1[0-2])-(0?[1-9]|[12]\d|3[01])$"
     match = re.match(date_pattern, str(date_string))  # 判断纯日期格式：2024-11-09
     if match is None:
@@ -125,15 +132,17 @@ class MysqlUpload:
 
         return wrapper
 
-    def dict_to_mysql(self, db_name, table_name, dict_data, main_key=None, unique_main_key=None, index_length=100):
+    @try_except
+    def dict_to_mysql(self, db_name, table_name, dict_data, icm_update=None, main_key=None, unique_main_key=None, index_length=100, set_type=None):
         """
         插入字典数据
         dict_data： 字典
-        main_key： 指定索引列
+        main_key： 指定索引列, 通常用日期列，默认会设置日期为索引
         unique_main_key： 指定唯一索引列
         index_length: 索引长度
+        icm_update: 增量更正，指定后 main_key 只用于检查/创建列，不能更新数据
+        set_type: {}
         """
-
         if not main_key:
             main_key = []
         if not unique_main_key:
@@ -167,7 +176,10 @@ class MysqlUpload:
                 print(f'创建 mysql 表: {table_name}')
 
             # 根据 dict_data 的值添加指定的数据类型
-            dtypes = self.cover_dict_dtypes(dict_data=dict_data)  # {'店铺名称': 'mediumtext',...}
+            dtypes, dict_data = self.cover_dict_dtypes(dict_data=dict_data)  # {'店铺名称': 'mediumtext',...}
+            if set_type:
+                dtypes.update(set_type)  # 自定义的数据类型
+
             # 检查列
             sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s;"
             cursor.execute(sql, (db_name, table_name))
@@ -181,59 +193,164 @@ class MysqlUpload:
                     cursor.execute(sql)
                     print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
 
-                    if col in main_key or col in unique_main_key:
-                        sql = f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = %s"
-                        cursor.execute(sql, (col))
-                        result = cursor.fetchone()  # 检查索引是否存在
-                        if not result:
-                            if col in main_key:
-                                cursor.execute(f"CREATE INDEX index_name ON `{table_name}`(`{col}`({index_length}))")
-                                print(f"创建索引列: {col}({dtypes[col]})")  # 添加列并指定数据类型
-                            elif col in unique_main_key:
-                                cursor.execute(f"CREATE UNIQUE INDEX index_name ON `{table_name}`(`{col}`({index_length}))")
-                                print(f"创建唯一索引: {col}({dtypes[col]})")  # 添加列并指定数据类型
+                    if col in main_key or col == '日期':
+                        sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
+                        print(f"设置为索引: {col}({dtypes[col]})")
+                        cursor.execute(sql)
+                    if col in unique_main_key:
+                        if dtypes[col] == 'mediumtext':
+                            sql = f"ALTER TABLE {table_name} ADD UNIQUE (`{col}`({index_length}))"
+                        else:
+                            sql = f"ALTER TABLE {table_name} ADD UNIQUE (`{col}`)"
+                        cursor.execute(sql)
+                    # if col in main_key or col in unique_main_key:
+                    #     sql = f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = %s"
+                    #     cursor.execute(sql, (col))
+                    #     result = cursor.fetchone()  # 检查索引是否存在
+                    #     if not result:
+                    #         if col in main_key:
+                    #             sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
+                    #             print(f"设置为索引: {col}({dtypes[col]})")
+                    #             cursor.execute(sql)
+                    #         elif col in unique_main_key:
+                    #             if dtypes[col] == 'mediumtext':
+                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`({index_length}));"
+                    #             else:
+                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`);"
+                    #             print(f"设置唯一索引: {col}({dtypes[col]})")
+                    #             print(sql)
+                    #             cursor.execute(sql)
             connection.commit()  # 提交事务
+            """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+            """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+            # 处理插入的数据
+            if icm_update:
+                """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列组合是数据表中唯一，值不会发生变化且不会重复，否则可能产生覆盖 """
+                sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                cursor.execute(sql, (db_name, {table_name}))
+                columns = cursor.fetchall()
+                cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+                update_col = [item for item in cols_exist if item not in icm_update and item != 'id']  # 除了主键外的其他列
+
+                # unique_keys 示例: `日期`, `余额`
+                unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
+                condition = []
+                for up_col in icm_update:
+                    condition += [f'`{up_col}` = "{dict_data[up_col]}"']
+                condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
+                sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
+                # print(sql)
+                # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                cursor.execute(sql)
+                results = cursor.fetchall()  # results 是数据库取出的数据
+                if results:  # 有数据返回，再进行增量检查
+                    for result in results:  # results 是数据库数据, dict_data 是传进来的数据
+                        change_col = []  # 发生变化的列名
+                        change_values = []  # 发生变化的数据
+                        for col in update_col:
+                            # 因为 mysql 里面有 decimal 数据类型，要移除末尾的 0 再做比较（df 默认将 5.00 小数截断为 5.0）
+                            df_value = str(dict_data[col])
+                            mysql_value = str(result[col])
+                            if '.' in df_value:
+                                df_value = re.sub(r'0+$', '', df_value)
+                                df_value = re.sub(r'\.$', '', df_value)
+                            if '.' in mysql_value:
+                                mysql_value = re.sub(r'0+$', '', mysql_value)
+                                mysql_value = re.sub(r'\.$', '', mysql_value)
+                            if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
+                                # print(f'{dict_data['日期']}{dict_data['商品id']}{col} 列的值有变化，{str(dict_data[col])}  !=  {str(result[col])}')
+                                change_values += [f"`{col}` = \"{str(dict_data[col])}\""]
+                                change_col.append(col)
+                        not_change_col = [item for item in update_col if item not in change_col]
+                        # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
+                        if change_values:  # change_values 有数据返回，表示值需要更新
+                            if not_change_col:
+                                not_change_values = [f'`{col}` = "{str(dict_data[col])}"' for col in not_change_col]
+                                not_change_values = ' AND '.join(
+                                    not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
+                                # print(change_values, not_change_values)
+                                condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
+                            change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
+                            sql = "UPDATE `%s` SET %s WHERE %s" % (table_name, change_values, condition)
+                            # print(sql)
+                            cursor.execute(sql)
+                else:  # 没有数据返回，则直接插入数据
+                    cols = ', '.join(f"`{item}`" for item in dict_data.keys())  # 列名需要转义
+                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                    values = ', '.join([f'"{item}"' for item in dict_data.values()])  # 值要加引号
+                    sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                    cursor.execute(sql)
+                connection.commit()  # 提交数据库
+                connection.close()
+                return
 
+            # 构建 keys
             keys_data = ', '.join([f'`{str(item)}`' for item in dict_data.keys()])
+            # 构建 values
             values_data = ', '.join(f'"{str(item)}"' for item in dict_data.values())
-            del dict_data['客户id']
-            update_datas = ', '.join([f'{k} = VALUES({k})' for k, v in dict_data.items()])
-            # print(update_datas)
+            # 构建其他键值，重复时要更新的其他键
+            if main_key:
+                for col in main_key:
+                    del dict_data[col]
+            if unique_main_key:
+                for col in unique_main_key:
+                    del dict_data[col]
+            # 涉及列名务必使用反引号
+            update_datas = ', '.join([f'`{k}` = VALUES(`{k}`)' for k, v in dict_data.items()])
+
+            # 构建 sql
             sql = f"INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s" % (table_name, keys_data, values_data, update_datas)
             # print(sql)
             cursor.execute(sql)
-            connection.commit()  # 提交事务
+            connection.commit()  # 提交数据库
         connection.close()
 
-
     def cover_dict_dtypes(self, dict_data):
         if not dict_data:
             print(f'mysql.py -> MysqlUpload -> cover_dict_dtypes -> 传入的字典不能为空')
             return
         __res_dict = {}
+        new_dict_data = {}
         for k, v in dict_data.items():
-            result1 = re.findall(r'商品编码|[商品宝贝]_?id|s[kp]u_?id|货号|款号|文件大小', str(k), re.IGNORECASE)
-            result2 = re.findall(r'占比|投产|产出|同比|环比|roi$', str(k), re.IGNORECASE)
-            date_type = is_valid_date(str(v))  # 判断日期时间
-            int_num = is_integer(str(v))  # 判断整数
-            count_int, count_float = count_decimal_places(str(v))  # 判断小数，返回小数位数
+            k = str(k).lower()
+            k = re.sub(r'[()\-，,$&~^、 （）\"\'“”=·/。》《><！!`]', '_', k, re.IGNORECASE)
+            k = k.replace('）', '')
+            k = re.sub(r'_{2,}', '_', k)
+            k = re.sub(r'_+$', '', k)
+            if str(v) == '':
+                v = 0
+            v = str(v)
+            v = re.sub('^-$|^--$|^nan$|^null$', '0', v, re.I)
+            v = re.sub(',|="|"', '', v, re.I)
+            if re.findall(r'^[-+]?\d+\.?\d*%$', v):
+                v = str(float(v.rstrip("%")) / 100)
+
+            result1 = re.findall(r'编码|_?id|货号|款号|文件大小', k, re.IGNORECASE)
+            result2 = re.findall(r'占比$|投产$|产出$|同比$|环比$|roi$|率$', k, re.IGNORECASE)
+            date_type = is_valid_date(v)  # 判断日期时间
+            int_num = is_integer(v)  # 判断整数
+            count_int, count_float = count_decimal_places(v)  # 判断小数，返回小数位数
             if result1:  # 京东sku/spu商品信息
                 __res_dict.update({k: 'mediumtext'})
-            elif result2:
-                __res_dict.update({k: 'decimal(10,2)'})
-            elif str(v) == '' or str(v).lower() == 'nan' or str(v).lower() == 'null':
-                v = 0
-                dict_data.update({k: v})
+            elif k == '日期':
+                __res_dict.update({k: 'DATE'})
+            elif k == '更新时间':
+                __res_dict.update({k: 'TIMESTAMP'})
+            elif str(v) == '':
                 __res_dict.update({k: 'mediumtext'})
-            elif date_type == 1:
+            elif result2:  # 小数
+                __res_dict.update({k: 'decimal(10,4)'})
+            elif date_type == 1:  # 纯日期
                 __res_dict.update({k: 'DATE'})
-            elif date_type == 2:
+            elif date_type == 2:  # 日期+时间
                 __res_dict.update({k: 'DATETIME'})
             elif int_num:
                 __res_dict.update({k: 'INT'})
             elif count_float > 0:
                 if count_int + count_float > 10:
-                    __res_dict.update({k: 'mediumtext'})
+                    if count_float > 5:
+                        v = round(float(v), 4)
+                    __res_dict.update({k: 'decimal(12,4)'})
                 elif count_float >= 6:
                     __res_dict.update({k: 'decimal(12,6)'})
                 elif count_float >= 4:
@@ -242,9 +359,8 @@ class MysqlUpload:
                     __res_dict.update({k: 'decimal(10,2)'})
             else:
                 __res_dict.update({k: 'mediumtext'})
-        return __res_dict
-
-
+            new_dict_data.update({k: v})
+        return __res_dict, new_dict_data
 
     @try_except
     def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'xigua_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, reset_id=False):
@@ -309,6 +425,11 @@ class MysqlUpload:
             for service_name, database in service_database.items():
                 # 2. 列数据类型转换，将 df 数据类型转换为 mysql 的数据类型
                 dtypes, cl, db_n, tb_n = self.convert_dtypes(df=df, db_name=db_name, table_name=table_name, path=json_path, service_name=service_name)
+                for dy in dtypes.keys():
+                    if '日期' == dy:
+                        dtypes.update({'日期': 'DATE'})
+                    if '更新时间' == dy:
+                        dtypes.update({'更新时间': 'TIMESTAMP'})
 
             #  有特殊字符不需转义
             sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s;"
@@ -425,9 +546,7 @@ class MysqlUpload:
                 # data 是传进来待处理的数据, 不是数据库数据
                 # data 示例: {'日期': Timestamp('2024-08-27 00:00:00'), '推广费余额': 33299, '品销宝余额': 2930.73, '短信剩余': 67471}
                 try:
-                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
-                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                    values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
+
                     condition = []
                     for k, v in data.items():
                         condition += [f'`{k}` = "{v}"']
@@ -1055,3 +1174,7 @@ if __name__ == '__main__':
     data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = data['username'], data['password'], data['host'], data['port']
     print(username, password, host, port)
+
+    ss = '2024-11-08'
+    ss= re.sub(r'\\N', '0', ss)
+    print(ss, '111')

{mdbq-2.9.2.dist-info → mdbq-2.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.9.2
+Version: 2.9.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.9.2.dist-info → mdbq-2.9.4.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=VtGP7KhUY-NUBA2CXt50hYuGOZ1bU_NeaBBvAUxYgTg,72338
+mdbq/aggregation/aggregation.py,sha256=IJS5ILEmYlrepj2oX6TDuMjab5rYEOpQuYyTpgfRbR0,73747
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=79uwiM2WqNNFxGpE2wKz742PRq-ZGgFjdOV0vgptHdY,3513
-mdbq/aggregation/query_data.py,sha256=iRgPljgOPE7dzhaaVxRXOEOOKQTmWg6sGsDplNLTvQw,100177
+mdbq/aggregation/query_data.py,sha256=w_p013oMdF6YovQwP6RY6wiPTKuuTfSn53Wo1RC_CD0,103372
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -18,7 +18,7 @@ mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
 mdbq/config/myconfig.py,sha256=EGymTlAimtHIDJ9egCtOehBEPOj6rea504kvsEZu64o,854
-mdbq/config/products.py,sha256=Sj4FSb2dZcMKp6ox-FJdIR87QLgMN_TJ7Z6KAWMTWyw,6214
+mdbq/config/products.py,sha256=sC4ctAiHR7ydkEXuIlvwvTPDLJXwengkG0hFWSQRFz0,6808
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
@@ -28,7 +28,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=jq2_lQFTnQR6N6QdSqxKqjennazITh5TdTg6j7oerYE,55006
+mdbq/mysql/mysql.py,sha256=5l5wFS6AFjXuZtb54JSnoqh_DlB3-tQd9LNac0CSYjQ,62538
 mdbq/mysql/recheck_mysql.py,sha256=jHQSlQy0PlQ_EYICQv_2nairUX3t6OIwPtSELKIpjkY,8702
 mdbq/mysql/s_query.py,sha256=bgNNIqYLDCHjD5KTFcm6x4u74selpAGs5ouJYuqX86k,8447
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -45,7 +45,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=jHrdGWBJQaSywx7V-U4YuM6vWkwC5SR5tTOOdB3YU_c,17306
-mdbq-2.9.2.dist-info/METADATA,sha256=3_u-jNgpFq2qX6uv2ufoYgPirGomwoz9IZkYmRp1BIc,243
-mdbq-2.9.2.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-2.9.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.9.2.dist-info/RECORD,,
+mdbq-2.9.4.dist-info/METADATA,sha256=1xFHayCMA4H3FNPMIEdut-xWBAWiwooZZrD6p7blLQU,243
+mdbq-2.9.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.9.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.9.4.dist-info/RECORD,,

{mdbq-2.9.2.dist-info → mdbq-2.9.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any