mdbq 2.9.3__py3-none-any.whl → 2.9.4__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1218,7 +1218,7 @@ def one_file_to_mysql(file, db_name, table_name):
 
 
 def test():
-    path = r'C:\同步空间\BaiduSyncdisk\原始文件3\天猫推广报表\品销宝'
+    path = r'/Users/xigua/Downloads/DMP报表'
 
     results = []
     for root, dirs, files in os.walk(path, topdown=False):
@@ -1226,12 +1226,16 @@ def test():
             if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
                 # df = pd.read_excel(os.path.join(root, name), header=0)
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                results.append(df)
                 # print(name)
-                # if len(df) == 0:
-                #     continue
-                # # df.insert(loc=1, column='店铺名称', value='万里马官方旗舰店')
-                df['更新时间'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-                df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                if len(df) == 0:
+                    continue
+                if '达摩盘消耗占比' in df.columns.tolist():
+                    print(name)
+                    df.pop('达摩盘消耗占比')
+                    # df.insert(loc=1, column='店铺名称', value='万里马官方旗舰店')
+                    # df['更新时间'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                    df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
                 # for col in ['更新时间']:
                 #     if col not in df.columns.tolist():
                 #         print(name)
@@ -1243,12 +1247,23 @@ def test():
                 # os.remove(os.path.join(root, name))
                 # results.append(df)
     # df = pd.concat(results)
-    # path = '/Users/xigua/Downloads/手淘搜索_本店引流词'
-    # filename = 'py_xg_手淘搜索_本店引流词_万里马官方旗舰店_2024-05_合并.csv'
+    # df.drop_duplicates(
+    #     subset=[
+    #         '日期',
+    #         '店铺名称',
+    #         '报表类型',
+    #         '消耗',
+    #         '展现量',
+    #         '点击量',
+    #     ], keep='last', inplace=True, ignore_index=True)
+    # df.fillna(0, inplace=True)
+    # for col in df.columns.tolist():
+    #     df[col] = df[col].apply(lambda x: 0 if str(x) == '' else x)
+    # path = '/Users/xigua/Downloads'
+    # filename = '品销宝_2024年_合并.csv'
     # df.to_csv(os.path.join(path, filename), encoding='utf-8_sig', index=False, header=True)
 
 
-
 if __name__ == '__main__':
     # username = 'root'
     # password = ''
@@ -1257,9 +1272,9 @@ if __name__ == '__main__':
     #
     # # 上传 1 个文件到数据库
     # one_file_to_mysql(
-    #     file=r'/Users/xi',
-    #     db_name='推广数据2',
-    #     table_name='超级直播',
+    #     file=r'/Users/xigua/Downloads/DMP报表_2024-10-23_2024-10-29.csv',
+    #     db_name='达摩盘3',
+    #     table_name='dmp人群报表',
     # )
 
     # test()
@@ -1267,10 +1282,10 @@ if __name__ == '__main__':
     col = 1
     if col:
         # 上传一个目录到指定数据库
-        db_name = '生意参谋3'
-        table_name = '商品排行'
+        db_name = '达摩盘3'
+        table_name = 'dmp人群报表'
         upload_dir(
-            path=r'C:\同步空间\BaiduSyncdisk\原始文件3\生意参谋\商品排行',
+            path=r'/Users/xigua/Downloads/DMP报表',
             db_name=db_name,
             collection_name=table_name,
         )

mdbq/aggregation/query_data.py

@@ -6,6 +6,7 @@ from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.aggregation import optimize_data
 from mdbq.config import myconfig
+from mdbq.config import products
 import datetime
 from dateutil.relativedelta import relativedelta
 import pandas as pd
@@ -15,6 +16,7 @@ import platform
 import getpass
 import json
 import os
+import time
 
 """
 程序用于下载数据库(调用 s_query.py 下载并清洗), 并对数据进行聚合清洗, 不会更新数据库信息;
@@ -855,8 +857,7 @@ class MysqlDatasQuery:
         df_crowd.sort_values('日期', ascending=True, ignore_index=True, inplace=True)
         df_crowd.drop_duplicates(subset=['人群id',], keep='last', inplace=True, ignore_index=True)
         df_crowd.pop('日期')
-        # print(df_crowd)
-
+        df_crowd = df_crowd.astype({'人群id': 'int64'}, errors='ignore')
         projection = {}
         df_dmp = self.download.data_to_df(
             db_name='达摩盘3',
@@ -865,6 +866,7 @@ class MysqlDatasQuery:
             end_date=end_date,
             projection=projection,
         )
+        df_dmp = df_dmp.astype({'人群id': 'int64'}, errors='ignore')
         df_dmp.sort_values('日期', ascending=True, ignore_index=True, inplace=True)
         df_dmp.drop_duplicates(subset=['日期', '人群id', '消耗_元'], keep='last', inplace=True, ignore_index=True)
         df = pd.merge(df_dmp, df_crowd, left_on=['人群id'], right_on=['人群id'], how='left')
@@ -1959,7 +1961,7 @@ class GroupBy:
         )
         return df
 
-    @try_except
+    # @try_except
     def performance_jd(self, jd_tg=True):
         jdtg, sku_sales = self.data_jdtg['京东_京准通'], self.data_jdtg['京东_sku_商品明细']
         jdtg = jdtg.groupby(['日期', '跟单sku_id'],
@@ -1973,6 +1975,7 @@ class GroupBy:
         df = df[['日期', '商品id', '货号', '成交单量', '成交金额', '成本价']]
         df['商品id'] = df['商品id'].astype(str)
         jdtg['跟单sku_id'] = jdtg['跟单sku_id'].astype(str)
+        jdtg = jdtg.astype({'日期': 'datetime64[ns]'}, errors='raise')
         if jd_tg is True:
             # 完整的数据表，包含全店所有推广、销售数据
             df = pd.merge(df, jdtg, how='left', left_on=['日期', '商品id'], right_on=['日期', '跟单sku_id'])  # df 合并推广表
@@ -2354,42 +2357,40 @@ def data_aggregation(months=1, is_juhe=True, less_dict=[]):
         )
 
 
-def main():
+def main(days=100, months=3):
     # 更新日期表
     date_table()
     # 更新货品年份基准表， 属性设置 3 - 货品年份基准
     p = products.Products()
     p.to_mysql()
 
+    # 数据聚合
+    data_aggregation(
+        months=months,
+        is_juhe=True,  # 生成聚合表
+        # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
+    )
+    time.sleep(60)
+
     system = platform.system()  # 本机系统
     host_name = socket.gethostname()  # 本机名
     conf = myconfig.main()
     db_list = conf[system][host_name]['mysql']['数据库集']
-    db_list = [item for item in db_list if item != '聚合数据']
-    # 清理所有非聚合数据的库
+    # 清理所有库
     optimize_data.op_data(
         db_name_lists=db_list,
-        days=5,
-        is_mongo=True,
-        is_mysql=True,
-    )
-
-    # 数据聚合
-    query_data.data_aggregation(months=3)
-    time.sleep(60)
-
-    # 清理聚合数据, mongodb 中没有聚合数据，所以只需要清理 mysql 即可
-    optimize_data.op_data(
-        db_name_lists=['聚合数据'],
-        days=100,
+        days=days,
         is_mongo=False,
         is_mysql=True,
     )
 
 
 if __name__ == '__main__':
-    data_aggregation(
-        months=3,
-        is_juhe=True,  # 生成聚合表
-        # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
-    )
+    main(days=100, months=3)
+
+    # data_aggregation(
+    #     months=3,
+    #     is_juhe=True,  # 生成聚合表
+    #     # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
+    # )
+

mdbq/config/products.py

@@ -124,25 +124,39 @@ class Products:
         ]
         self.datas += my_datas
 
+
     def to_mysql(self):
         self.update_my_datas()
         df = pd.DataFrame(self.datas)
-        m = mysql.MysqlUpload(
+        m_engine = mysql.MysqlUpload(
             username=username,
             password=password,
             host=host,
             port=port,
         )
-        m.df_to_mysql(
-            df=df,
-            db_name='属性设置3',
-            table_name='货品年份基准',
-            move_insert = False,
-            df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
-            drop_duplicates=True,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
-            icm_update=[],
-            service_database=service_database,  # 用来追踪处理进度
-        )
+        for dict_data in df.to_dict(orient='records'):
+            m_engine.dict_to_mysql(
+                db_name='属性设置3',
+                table_name='货品年份基准',
+                dict_data=dict_data,
+                # icm_update=['日期', '店铺名称', '宝贝id'],  # 唯一组合键
+                unique_main_key=['商品id'],
+                set_type={
+                    '商品id': 'mediumtext',
+                    '平台': 'mediumtext',
+                    '上市年份': 'mediumtext',
+                },
+            )
+        # m.df_to_mysql(
+        #     df=df,
+        #     db_name='属性设置3',
+        #     table_name='货品年份基准',
+        #     move_insert = False,
+        #     df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+        #     drop_duplicates=True,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+        #     icm_update=[],
+        #     service_database=service_database,  # 用来追踪处理进度
+        # )
 
     def market_date(self, product_id: int):
         try:

mdbq/mysql/mysql.py

@@ -132,6 +132,7 @@ class MysqlUpload:
 
         return wrapper
 
+    @try_except
     def dict_to_mysql(self, db_name, table_name, dict_data, icm_update=None, main_key=None, unique_main_key=None, index_length=100, set_type=None):
         """
         插入字典数据
@@ -294,10 +295,12 @@ class MysqlUpload:
             if unique_main_key:
                 for col in unique_main_key:
                     del dict_data[col]
-            update_datas = ', '.join([f'{k} = VALUES({k})' for k, v in dict_data.items()])
+            # 涉及列名务必使用反引号
+            update_datas = ', '.join([f'`{k}` = VALUES(`{k}`)' for k, v in dict_data.items()])
 
             # 构建 sql
             sql = f"INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s" % (table_name, keys_data, values_data, update_datas)
+            # print(sql)
             cursor.execute(sql)
             connection.commit()  # 提交数据库
         connection.close()
@@ -329,6 +332,10 @@ class MysqlUpload:
             count_int, count_float = count_decimal_places(v)  # 判断小数，返回小数位数
             if result1:  # 京东sku/spu商品信息
                 __res_dict.update({k: 'mediumtext'})
+            elif k == '日期':
+                __res_dict.update({k: 'DATE'})
+            elif k == '更新时间':
+                __res_dict.update({k: 'TIMESTAMP'})
             elif str(v) == '':
                 __res_dict.update({k: 'mediumtext'})
             elif result2:  # 小数
@@ -418,6 +425,11 @@ class MysqlUpload:
             for service_name, database in service_database.items():
                 # 2. 列数据类型转换，将 df 数据类型转换为 mysql 的数据类型
                 dtypes, cl, db_n, tb_n = self.convert_dtypes(df=df, db_name=db_name, table_name=table_name, path=json_path, service_name=service_name)
+                for dy in dtypes.keys():
+                    if '日期' == dy:
+                        dtypes.update({'日期': 'DATE'})
+                    if '更新时间' == dy:
+                        dtypes.update({'更新时间': 'TIMESTAMP'})
 
             #  有特殊字符不需转义
             sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s;"

{mdbq-2.9.3.dist-info → mdbq-2.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.9.3
+Version: 2.9.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.9.3.dist-info → mdbq-2.9.4.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=IVh9SFO1yp12qDBuEOWTi9SAytYktKBrsPJNPuDetSM,73254
+mdbq/aggregation/aggregation.py,sha256=IJS5ILEmYlrepj2oX6TDuMjab5rYEOpQuYyTpgfRbR0,73747
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=79uwiM2WqNNFxGpE2wKz742PRq-ZGgFjdOV0vgptHdY,3513
-mdbq/aggregation/query_data.py,sha256=zut8WyyAKTULfGWMltyQYqsVsIaBDUU8E3w2_UL4hbA,103248
+mdbq/aggregation/query_data.py,sha256=w_p013oMdF6YovQwP6RY6wiPTKuuTfSn53Wo1RC_CD0,103372
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -18,7 +18,7 @@ mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
 mdbq/config/myconfig.py,sha256=EGymTlAimtHIDJ9egCtOehBEPOj6rea504kvsEZu64o,854
-mdbq/config/products.py,sha256=Sj4FSb2dZcMKp6ox-FJdIR87QLgMN_TJ7Z6KAWMTWyw,6214
+mdbq/config/products.py,sha256=sC4ctAiHR7ydkEXuIlvwvTPDLJXwengkG0hFWSQRFz0,6808
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
@@ -28,7 +28,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=tKkgjbOvy5uIn7Z-ws_biS-04-UHnr5rKqNvtWr_Yss,62024
+mdbq/mysql/mysql.py,sha256=5l5wFS6AFjXuZtb54JSnoqh_DlB3-tQd9LNac0CSYjQ,62538
 mdbq/mysql/recheck_mysql.py,sha256=jHQSlQy0PlQ_EYICQv_2nairUX3t6OIwPtSELKIpjkY,8702
 mdbq/mysql/s_query.py,sha256=bgNNIqYLDCHjD5KTFcm6x4u74selpAGs5ouJYuqX86k,8447
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -45,7 +45,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=jHrdGWBJQaSywx7V-U4YuM6vWkwC5SR5tTOOdB3YU_c,17306
-mdbq-2.9.3.dist-info/METADATA,sha256=fL1JR-lJNlMr2cIzQIEO460TetP9yzBerJPCJYnBRQ8,243
-mdbq-2.9.3.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-2.9.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.9.3.dist-info/RECORD,,
+mdbq-2.9.4.dist-info/METADATA,sha256=1xFHayCMA4H3FNPMIEdut-xWBAWiwooZZrD6p7blLQU,243
+mdbq-2.9.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.9.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.9.4.dist-info/RECORD,,

{mdbq-2.9.3.dist-info → mdbq-2.9.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any