mdbq 2.4.5__py3-none-any.whl → 2.4.7__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -260,6 +260,7 @@ class DatabaseUpdate:
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
                         check_remove_file = True
+                        os.remove(os.path.join(root, name))
                         continue
                     df.replace(to_replace=[','], value='', regex=True, inplace=True)
                     df.insert(loc=0, column='日期', value=pattern[0][1])
@@ -398,7 +399,7 @@ class DatabaseUpdate:
                     pattern = re.findall(r'(.*[\u4e00-\u9fa5])(\d{4})(\d{2})(\d{2})\.', name)
                     if not pattern or '省份城市分析2' not in name:
                         print(f'{name}  不支持或已转换的表格')
-                        # os.remove(os.path.join(root, name))  # 直接删掉，避免被分到原始文件, encoding 不同会引发错误
+                        os.remove(os.path.join(root, name))  # 直接删掉，避免被分到原始文件, encoding 不同会引发错误
                         check_remove_file = True
                         continue
                     date = '-'.join(pattern[0][1:])
@@ -406,6 +407,7 @@ class DatabaseUpdate:
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
                         check_remove_file = True
+                        os.remove(os.path.join(root, name))
                         continue
                     df['省'] = df['省份'].apply(lambda x: x if ' ├─ ' not in x and ' └─ ' not in x else None)
                     df['城市'] = df[['省份', '省']].apply(lambda x: '汇总' if x['省'] else x['省份'], axis=1)
@@ -1291,7 +1293,7 @@ def test2():
     dp.cleaning(is_move=False, )  # 清洗数据, 存入 self.datas
     dp.upload_df(service_databases=[
         # {'home_lx': 'mongodb'},
-        {'home_lx': 'mysql'},
+        {'company': 'mysql'},
         # {'nas': 'mysql'}
     ], path=None, service_name=None)
 

mdbq/aggregation/query_data.py

@@ -998,6 +998,29 @@ class GroupBy:
             # df = df.head(1000)
             # df.to_csv('/Users/xigua/Downloads/test.csv', index=False, header=True, encoding='utf-8_sig')
             # breakpoint()
+
+            # 下面是添加人群 AIPL 分类
+            dir_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\分类配置文件.xlsx'
+            dir_file2 = '/Volumes/时尚事业部/01.运营部/0-电商周报-每周五更新/分类配置文件.xlsx'
+            if not os.path.isfile(dir_file):
+                dir_file = dir_file2
+            if os.path.isfile(dir_file):
+                df_fl = pd.read_excel(dir_file, sheet_name='人群分类', header=0)
+                df_fl = df_fl[['人群名字', '人群分类']]
+                # 合并并获取分类信息
+                df = pd.merge(df, df_fl, left_on=['人群名字'], right_on=['人群名字'], how='left')
+                df['人群分类'].fillna('', inplace=True)
+            if '人群分类' in df.columns.tolist():
+                # 这行决定了，从文件中读取的分类信息优先级高于内部函数的分类规则
+                df['人群分类'] = df.apply(
+                    lambda x: self.set_crowd(keyword=str(x['人群名字']), as_file=False) if x['人群分类'] == ''
+                    else x['人群分类'], axis=1
+                )
+            else:
+                df['人群分类'] = df['人群名字'].apply(lambda x: self.set_crowd(keyword=str(x), as_file=False))
+            df['人群分类'] = df['人群分类'].apply(lambda x: str(x).upper() if x else x)
+            # df.to_csv('/Users/xigua/Downloads/test_人群分类.csv', index=False, header=True, encoding='utf-8_sig')
+            # breakpoint()
             return df
 
         elif '天猫_关键词报表' in table_name:
@@ -1620,6 +1643,41 @@ class GroupBy:
                 break
         return result
 
+    def set_crowd(self, keyword, as_file=False):
+        """ 推广人群报表，人群分类， """
+        result_a = re.findall('_a$|_a_|_ai|^a_', str(keyword), re.IGNORECASE)
+        result_i = re.findall('_i$|_i_|^i_', str(keyword), re.IGNORECASE)
+        result_p = re.findall('_p$|_p_|_pl|^p_||^pl_', str(keyword), re.IGNORECASE)
+        result_l = re.findall('_l$|_l_|^l_', str(keyword), re.IGNORECASE)
+
+        datas = [
+            {
+                '类别': 'A',
+                '值': result_a,
+            },
+            {
+                '类别': 'I',
+                '值': result_i,
+            },
+            {
+                '类别': 'P',
+                '值': result_p,
+            },
+            {
+                '类别': 'L',
+                '值': result_l,
+            }
+        ]
+
+        is_res = False
+        for data in datas:
+            if data['值']:
+                data['值'] = [item for item in data['值'] if item != '']
+                if data['值']:
+                    return data['类别']
+        if not is_res:
+            return ''
+
     # @try_except
     def performance(self, bb_tg=True):
          # print(self.data_tgyj)
@@ -1870,7 +1928,7 @@ def data_aggregation_one(service_databases=[{}], months=1):
                 )  # 3. 回传数据库
 
 
-def data_aggregation(service_databases=[{}], months=1):
+def data_aggregation(service_databases=[{}], months=1, is_juhe=True):
     """
     1. 从数据库中读取数据
     2. 数据聚合清洗
@@ -2054,54 +2112,51 @@ def data_aggregation(service_databases=[{}], months=1):
                         icm_update=unique_key_list,
                         service_database=service_database,
                     )  # 3. 回传数据库
-            res = g.performance(bb_tg=True)   # 盈亏表，依赖其他表，单独做
-            m.df_to_mysql(
-                df=res,
-                db_name='聚合数据',
-                table_name='_全店商品销售',
-                move_insert=True,  # 先删除，再插入
-                # df_sql=True,
-                # drop_duplicates=False,
-                # icm_update=['日期', '商品id'],  # 设置唯一主键
-                service_database=service_database,
-            )
-            res = g.performance(bb_tg=False)  # 盈亏表，依赖其他表，单独做
-            m.df_to_mysql(
-                df=res,
-                db_name='聚合数据',
-                table_name='_推广商品销售',
-                move_insert=True,  # 先删除，再插入
-                # df_sql=True,
-                # drop_duplicates=False,
-                # icm_update=['日期', '商品id'],  # 设置唯一主键
-                service_database=service_database,
-            )
-
-            res = g.performance_concat(bb_tg=False)  # 推广主体合并直播表，依赖其他表，单独做
-            m.df_to_mysql(
-                df=res,
-                db_name='聚合数据',
-                table_name='天猫_推广汇总',
-                move_insert=True,  # 先删除，再插入
-                # df_sql=True,
-                # drop_duplicates=False,
-                # icm_update=['日期', '推广渠道', '营销场景', '商品id', '花费', '展现量', '点击量'],  # 设置唯一主键
-                service_database=service_database,
-            )
-
-
-            res = g.performance_jd(jd_tg=False)  # 盈亏表，依赖其他表，单独做
-            m.df_to_mysql(
-                df=res,
-                db_name='聚合数据',
-                table_name='_京东_推广商品销售',
-                move_insert=True,  # 先删除，再插入
-                # df_sql=True,
-                # drop_duplicates=False,
-                # icm_update=['日期', '跟单sku id', '货号', '花费'],  # 设置唯一主键
-                service_database=service_database,
-            )
-
+            if is_juhe:
+                res = g.performance(bb_tg=True)   # 盈亏表，依赖其他表，单独做
+                m.df_to_mysql(
+                    df=res,
+                    db_name='聚合数据',
+                    table_name='_全店商品销售',
+                    move_insert=True,  # 先删除，再插入
+                    # df_sql=True,
+                    # drop_duplicates=False,
+                    # icm_update=['日期', '商品id'],  # 设置唯一主键
+                    service_database=service_database,
+                )
+                res = g.performance(bb_tg=False)  # 盈亏表，依赖其他表，单独做
+                m.df_to_mysql(
+                    df=res,
+                    db_name='聚合数据',
+                    table_name='_推广商品销售',
+                    move_insert=True,  # 先删除，再插入
+                    # df_sql=True,
+                    # drop_duplicates=False,
+                    # icm_update=['日期', '商品id'],  # 设置唯一主键
+                    service_database=service_database,
+                )
+                res = g.performance_concat(bb_tg=False)  # 推广主体合并直播表，依赖其他表，单独做
+                m.df_to_mysql(
+                    df=res,
+                    db_name='聚合数据',
+                    table_name='天猫_推广汇总',
+                    move_insert=True,  # 先删除，再插入
+                    # df_sql=True,
+                    # drop_duplicates=False,
+                    # icm_update=['日期', '推广渠道', '营销场景', '商品id', '花费', '展现量', '点击量'],  # 设置唯一主键
+                    service_database=service_database,
+                )
+                res = g.performance_jd(jd_tg=False)  # 盈亏表，依赖其他表，单独做
+                m.df_to_mysql(
+                    df=res,
+                    db_name='聚合数据',
+                    table_name='_京东_推广商品销售',
+                    move_insert=True,  # 先删除，再插入
+                    # df_sql=True,
+                    # drop_duplicates=False,
+                    # icm_update=['日期', '跟单sku id', '货号', '花费'],  # 设置唯一主键
+                    service_database=service_database,
+                )
 
     # 这里要注释掉，不然 copysh.py 可能有问题，这里主要修改配置文件，后续触发 home_lx 的 optimize_datas.py(有s)程序进行全局清理
     # optimize_data.op_data(service_databases=service_databases, days=3650)  # 立即启动对聚合数据的清理工作
@@ -2112,6 +2167,6 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=0)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=0, is_juhe=False)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

mdbq/mysql/mysql.py

@@ -167,7 +167,7 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
-                print(f'重置自增')
+                # print(f'重置自增')
                 # 6. 重置自增列
                 try:
                     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")

{mdbq-2.4.5.dist-info → mdbq-2.4.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.4.5
+Version: 2.4.7
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.4.5.dist-info → mdbq-2.4.7.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=kHKCBRMfpv0F1dq4n75LqOt44K4lRWQCt8MeM6n6hnM,76013
+mdbq/aggregation/aggregation.py,sha256=nPp5fOLktxejNEak3SyTnKLjwzK1l2xjbV45X-I4LFQ,76131
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=Wis40oL04M7E1pkvgNPjyVFAUe-zgjimjIVAikxYY8Y,4418
-mdbq/aggregation/query_data.py,sha256=b-beZv7JHMUe7jK0wr-cNClMVYOop3meb1fAGSadfIQ,96001
+mdbq/aggregation/query_data.py,sha256=_RYPlUDg5Hmfm663gx7vGh0HHfgnik9BDqLPQCl9w5A,98932
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -25,7 +25,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=D_0KO7JGFId9brcC6UsHKPgmKx7JMJLd68fqCh8XsSw,46848
+mdbq/mysql/mysql.py,sha256=ZJ5hrwB-oJgvfcTI9Y2E_vwmCNgbxmiU0AJLeCfQFJ8,46850
 mdbq/mysql/s_query.py,sha256=37GGHzRpycfUjsYEoQgDpdEs9JwjW-LxFXnGwwP2b2Q,8403
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -41,7 +41,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=Olq7IJP9itM4wuNxZeHOG-Q3i8wWyB4hY8TUGGwCvQ0,14104
-mdbq-2.4.5.dist-info/METADATA,sha256=Viga-WzhqgOumbM49YE8NbTlKMQ-L-vdJkDSueMeI3E,245
-mdbq-2.4.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.4.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.4.5.dist-info/RECORD,,
+mdbq-2.4.7.dist-info/METADATA,sha256=dBLtWhjOK1TZ87LrmoRgLAz7NHJYtQiC01dRimYwZ90,245
+mdbq-2.4.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.4.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.4.7.dist-info/RECORD,,