mdbq 1.2.5__py3-none-any.whl → 1.2.6__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -850,6 +850,8 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if len(df) == 0:
                         continue
+                    # if '新版' not in name:
+                    #     continue
                     cv = converter.DataFrameConverter()
                     df = cv.convert_df_cols(df=df)  # 清理列名和 df 中的非法字符
                     try:
@@ -973,5 +975,13 @@ if __name__ == '__main__':
     #     target_service='home_lx',
     #     database='mysql'
     # )
+    # db_name = '生意参谋2'
+    # table_name = '店铺来源_日数据_新版'
+    # upload_dir(
+    #     path='/Users/xigua/数据中心/原始文件2/生意参谋/流量来源',
+    #     db_name=db_name,
+    #     collection_name=table_name,
+    #     dbs={'mysql': True, 'mongodb': False},
+    # )
 
     test2()

mdbq/aggregation/query_data.py

@@ -240,6 +240,7 @@ class GroupBy:
                 '总成交金额': '成交金额'
             }, inplace=True)
             df = df.astype({
+                '商品id': str,
                 '花费': float,
                 '展现量': int,
                 '点击量': int,
@@ -263,12 +264,13 @@ class GroupBy:
                 )
             else:
                 df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
-                    **{'加购量': ('加购量', np.min),
-                       '成交笔数': ('成交笔数', np.min),
-                       '成交金额': ('成交金额', np.min),
-                       '自然流量曝光量': ('自然流量曝光量', np.min),
-                       '直接成交笔数': ('直接成交笔数', np.max),
-                       '直接成交金额': ('直接成交金额', np.max)
+                    **{
+                        '加购量': ('加购量', np.min),
+                        '成交笔数': ('成交笔数', np.min),
+                        '成交金额': ('成交金额', np.min),
+                        '自然流量曝光量': ('自然流量曝光量', np.min),
+                        '直接成交笔数': ('直接成交笔数', np.max),
+                        '直接成交金额': ('直接成交金额', np.max)
                        }
                 )
             df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
@@ -276,10 +278,10 @@ class GroupBy:
                     **{
                         '花费': ('花费', np.sum),
                         '成交笔数': ('成交笔数', np.max),
-                       '成交金额': ('成交金额', np.max),
-                       '自然流量曝光量': ('自然流量曝光量', np.max),
-                       '直接成交笔数': ('直接成交笔数', np.max),
-                       '直接成交金额': ('直接成交金额', np.max)
+                        '成交金额': ('成交金额', np.max),
+                        '自然流量曝光量': ('自然流量曝光量', np.max),
+                        '直接成交笔数': ('直接成交笔数', np.max),
+                        '直接成交金额': ('直接成交金额', np.max)
                        }
                 )
             self.data_tgyj.update(
@@ -290,6 +292,7 @@ class GroupBy:
             return df
         elif '宝贝指标' in table_name:
             """ 聚合时不可以加商家编码，编码有些是空白，有些是 0 """
+            df['宝贝id'] = df['宝贝id'].astype(str)
             df.fillna(0, inplace=True)
             # df = df[(df['销售额'] != 0) | (df['退款额'] != 0)]  # 注释掉, 因为后续使用生意经作为基准合并推广表，需确保所有商品id 齐全
             df = df.groupby(['日期', '宝贝id', '行业类目'], as_index=False).agg(
@@ -320,6 +323,7 @@ class GroupBy:
         elif '店铺来源_日数据' in table_name:
             return df
         elif '商品id编码表' in table_name:
+            df['宝贝id'] = df['宝贝id'].astype(str)
             df.drop_duplicates(subset='宝贝id', keep='last', inplace=True, ignore_index=True)
             # df['行业类目'] = df['行业类目'].apply(lambda x: re.sub(' ', '', x))
             try:
@@ -359,6 +363,7 @@ class GroupBy:
                     table_name: df[['商品id', '商品图片']],
                 }
             )
+            df['商品id'] = df['商品id'].astype(str)
             return df
         elif '商品成本' in table_name:
             df.sort_values(by=['款号', '日期'], ascending=[False, True], ignore_index=True, inplace=True)
@@ -373,7 +378,7 @@ class GroupBy:
             print(f'<{table_name}>: Groupby 类尚未配置，数据为空')
             return pd.DataFrame({})
 
-    @try_except
+    # @try_except
     def performance(self, bb_tg=True):
          # print(self.data_tgyj)
         tg, syj, idbm, pic, cost = (
@@ -390,13 +395,14 @@ class GroupBy:
         df = pd.merge(tg, df, how='left', left_on='商品id', right_on='宝贝id')
         df.drop(labels='宝贝id', axis=1, inplace=True)
         if bb_tg is True:
-             # 生意经合并推广表，完整的数据表，包含全店所有推广、销售数据
+            # 生意经合并推广表，完整的数据表，包含全店所有推广、销售数据
             df = pd.merge(syj, df, how='left', left_on=['日期', '宝贝id'], right_on=['日期', '商品id'])
         else:
             # 推广表合并生意经 , 以推广数据为基准，销售数据不齐全
             df = pd.merge(df, syj, how='left', left_on=['日期', '商品id'], right_on=['日期', '宝贝id'])
         df.drop(labels='宝贝id', axis=1, inplace=True)
         df.drop_duplicates(subset=['日期', '商品id', '花费', '销售额'], keep='last', inplace=True, ignore_index=True)
+        df['成本价'] = df['成本价'].astype('float64')
         df['商品成本'] = df.apply(lambda x: (x['成本价'] + x['销售额']/x['销售量'] * 0.11 + 6) * x['销售量'] if x['销售量'] > 0 else 0, axis=1)
         df['商品毛利'] = df.apply(lambda x: x['销售额'] - x['商品成本'], axis=1)
         df['毛利率'] = df.apply(lambda x: round((x['销售额'] - x['商品成本']) / x['销售额'], 4) if x['销售额'] > 0 else 0, axis=1)
@@ -493,12 +499,13 @@ def data_aggregation(service_databases=[{}]):
     for service_database in service_databases:
         for service_name, database in service_database.items():
             sdq = MysqlDatasQuery(target_service=service_name)  # 实例化数据处理类
-            sdq.months = 1  # 设置数据周期， 1 表示近 2 个月
+            sdq.months = 0  # 设置数据周期， 1 表示近 2 个月
             g = GroupBy()  # 实例化数据聚合类
             # 实例化数据库连接
             username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
             m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
 
+            # 从数据库中获取数据, 返回包含 df 数据的字典
             data_dict = [
                 {
                     '数据库名': '聚合数据',
@@ -531,19 +538,19 @@ def data_aggregation(service_databases=[{}]):
                     '数据主体': sdq.sp_cost(),
                 },
             ]
-            for items in data_dict:
+            for items in data_dict:  # 遍历返回结果
                 db_name, table_name, df = items['数据库名'], items['集合名'], items['数据主体']
                 df = g.groupby(df=df, table_name=table_name, is_maximize=True)  # 2. 聚合数据
                 # g.as_csv(df=df, filename=table_name + '.csv')
-                m.df_to_mysql(df=df, db_name=db_name, table_name=table_name)  # 3. 回传数据库
+                m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, drop_dup=True)  # 3. 回传数据库
             res = g.performance(bb_tg=True)   # 盈亏表，依赖其他表，单独做
-            m.df_to_mysql(df=res, db_name='聚合数据', table_name='_全店商品销售')
+            m.df_to_mysql(df=res, db_name='聚合数据', table_name='_全店商品销售', drop_dup=True)
             res = g.performance(bb_tg=False)  # 盈亏表，依赖其他表，单独做
-            m.df_to_mysql(df=res, db_name='聚合数据', table_name='_推广商品销售')
+            m.df_to_mysql(df=res, db_name='聚合数据', table_name='_推广商品销售', drop_dup=True)
 
     # optimize_data.op_data(service_databases=service_databases, days=3650)  # 立即启动对聚合数据的清理工作
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}])
+    data_aggregation(service_databases=[{'home_lx': 'mysql'}])
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

mdbq/mysql/mysql.py

@@ -65,7 +65,7 @@ class MysqlUpload:
         将 df 写入数据库
         db_name: 数据库名称
         table_name: 集合/表名称
-        df_sql: 使用 df.to_sql 函数上传整个表
+        df_sql: 使用 df.to_sql 函数上传整个表, 不会排重
         drop_duplicates：仅限于聚合数据使用，其他情况不要设置此参数
         drop_dup: 值为 True 时检查重复数据再插入，反之直接上传
         filename: 传这个参数是方便定位产生错误的文件
@@ -153,7 +153,7 @@ class MysqlUpload:
 
             if df_sql:
                 now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-                print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count},{self.filename}')
+                print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {self.filename}')
                 engine = create_engine(
                     f"mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}")  # 创建数据库引擎
                 df.to_sql(
@@ -186,7 +186,7 @@ class MysqlUpload:
 
             # 5. 更新插入数据
             now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count},{self.filename}')
+            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {self.filename}')
 
             datas = df.to_dict(orient='records')
             for data in datas:

mdbq/mysql/s_query.py

@@ -12,6 +12,7 @@ from sqlalchemy import create_engine
 import os
 import calendar
 from mdbq.config import get_myconf
+from mdbq.dataframe import converter
 
 warnings.filterwarnings('ignore')
 """
@@ -84,6 +85,8 @@ class QueryDatas:
 
         if len(df) == 0:
             print(f'database: {db_name}, table: {table_name} 查询的数据为空')
+        cv = converter.DataFrameConverter()
+        df = cv.convert_df_cols(df)
         return df
 
     def columns_to_list(self, db_name, table_name,  columns_name) -> list:

{mdbq-1.2.5.dist-info → mdbq-1.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.2.5
+Version: 1.2.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.2.5.dist-info → mdbq-1.2.6.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=UKpV2PrazABh5dlOcmmhs6vfgMYxo9sk05SAn-fizSw,55402
+mdbq/aggregation/aggregation.py,sha256=mBgIY7afloW8H5qoBy56vCabIQRxVvAhrRZgGbZUxFQ,55791
 mdbq/aggregation/df_types.py,sha256=rHLIgv82PJSFmDvXkZyOJAffXkFyyMyFO23w9tUt8EQ,7525
 mdbq/aggregation/mysql_types.py,sha256=umVixmbFZM63k-QhVWLvOuhcAde4P_oDKbdo8ry2O9w,10633
 mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
-mdbq/aggregation/query_data.py,sha256=s1QjZIr0ZODgiqbH6Rh-7jtTY5JJXpx7o5lgE32rGqE,25249
+mdbq/aggregation/query_data.py,sha256=fg_9OdNSwHbo9vhK1pAKOazHFHZfE9_rBxRyQIWJX9U,25694
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -24,8 +24,8 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=9lAUY6-AcLctvoqa7qFsNtWxBA4DBFNDzdkLJCkbr48,37528
-mdbq/mysql/s_query.py,sha256=4c24SwbqtnO33o8CgWlTQ_j8sZYl5BRIQkaD9CI-vTY,7901
+mdbq/mysql/mysql.py,sha256=KvUQflP5sYOECTHOs2Fs9ABcQvgPCbBnAX2ZlE3JjgY,37544
+mdbq/mysql/s_query.py,sha256=a33aYhW6gAnspIZfQ7l23ePln9-MD1f_ukypr5M0jd8,8018
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
@@ -35,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.2.5.dist-info/METADATA,sha256=yz9ZMV0ZFZti3QcBY4w-eOocIVFRitvrGvXqMnWhcys,245
-mdbq-1.2.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-1.2.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.2.5.dist-info/RECORD,,
+mdbq-1.2.6.dist-info/METADATA,sha256=_s1z5j_Q_dSi4lrw46NcpwMlgz5TkZnndOmWp4290Mk,245
+mdbq-1.2.6.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.2.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.2.6.dist-info/RECORD,,

{mdbq-1.2.5.dist-info → mdbq-1.2.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any