mdbq 2.8.3__py3-none-any.whl → 2.8.5__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1201,8 +1201,8 @@ def one_file_to_mysql(file, db_name, table_name):
 
 
 def test():
-    path = os.path.relpath(r'/Users/xigua/Downloads/直播间')
-    results = []
+    path = os.path.relpath(r'/Users/xigua/Downloads/未命名文件夹')
+
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
@@ -1212,43 +1212,35 @@ def test():
                 # print(name)
                 if len(df) == 0:
                     continue
-                results.append(df)
-                # df = df[df['日期'] != '']
+                # df.insert(loc=1, column='店铺名称', value='万里马官方旗舰店')
+                if '颜色编码' in df.columns.tolist():
+                    print(name)
+                    df.pop('颜色编码')
+                    df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                # pattern = re.findall(r'\d{4}-\d{2}-\d{2}_\d{4}-\d{2}-\d{2}', name)[0]
+                # new_name = f'py_xg_店铺销售指标_万里马官方旗舰店_{pattern}.csv'
                 # df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
-    df = pd.concat(results)
-
-    df.rename(columns={
-        '订单id': '计划ID',
-        '投资回报率': '投入产出比',
-        '直接投资回报率': '直接成交投入产出比',
-        '预售金额': '直接成交投入产出比',
-        '直接投资回报率': '总预售成交金额',
-        '预售订单量': '总预售成交笔数',
-        # '商品点击量': '展现量',
-        # '商品点击率': '观看率',
-    }, inplace=True)
-
-    df.to_csv(os.path.join('/Users/xigua/Downloads/', f'py_xg_tg_report_超级直播报表_人群_万里马官方旗舰店_.csv'), encoding='utf-8_sig', index=False, header=True)
+                # os.remove(os.path.join(root, name))
 
 
 if __name__ == '__main__':
-    username = 'root'
-    password = ''
-    host = ''
-    port = ''
-
-    # 上传 1 个文件到数据库
-    one_file_to_mysql(
-        file=r'/Users/xi',
-        db_name='推广数据2',
-        table_name='超级直播',
-    )
+    # username = 'root'
+    # password = ''
+    # host = ''
+    # port = ''
+    #
+    # # 上传 1 个文件到数据库
+    # one_file_to_mysql(
+    #     file=r'/Users/xi',
+    #     db_name='推广数据2',
+    #     table_name='超级直播',
+    # )
 
     # # 上传一个目录到指定数据库
-    # db_name = '推广数据2'
-    # table_name = '超级直播'
+    # db_name = '生意经3'
+    # table_name = '宝贝指标'
     # upload_dir(
-    #     path=os.path.relpath(r'/Users/xigua/数据中心/原始文件3/天猫推广报表/超级直播报表_人群/2024-04'),
+    #     path=os.path.relpath(r'/Users/xigua/数据中心/原始文件3/生意经/宝贝指标sdff'),
     #     db_name=db_name,
     #     collection_name=table_name,
     # )

mdbq/aggregation/mysql_types.py

@@ -64,7 +64,7 @@ class DataTypes:
     def get_mysql_types(self, cl, dtypes, db_name, table_name, is_file_dtype=True):
         """ 更新 mysql 的 types 信息到 json 文件 """
         if cl in self.datas.keys():
-            if db_name in list(self.datas[cl].keys()):  # ['京东数据2', '推广数据2', '生意参谋2', '生意经2']
+            if db_name in list(self.datas[cl].keys()):  # ['京东数据2', '推广数据2', '生意参谋2', '生意经3']
                 if table_name in list(self.datas[cl][db_name].keys()):
                     if is_file_dtype:  # 旧数据优先
                         # # 用 dtypes 更新, 允许手动指定 json 文件里面的数据类型
@@ -182,7 +182,7 @@ def mysql_all_dtypes(db_name=None, table_name=None, path=None):
     sys_lists = ['information_schema', 'mysql', 'performance_schema', 'sakila', 'sys']
     db_name_lists = [item for item in db_name_lists if item not in sys_lists]
 
-    results = []  # 返回结果示例: [{'云电影': '电影更新'}, {'生意经2': 'e3_零售明细统计'}]
+    results = []  # 返回结果示例: [{'云电影': '电影更新'}, {'生意经3': 'e3_零售明细统计'}]
     for db_ in db_name_lists:
         config.update({'database': db_})  # 添加更新 config 字段
         connection = pymysql.connect(**config)  # 连接数据库

mdbq/aggregation/query_data.py

@@ -160,7 +160,7 @@ class MysqlDatasQuery:
             '退货量_发货后': 1,
         }
         df = self.download.data_to_df(
-            db_name='生意经2',
+            db_name='生意经3',
             table_name='宝贝指标',
             start_date=start_date,
             end_date=end_date,
@@ -283,7 +283,7 @@ class MysqlDatasQuery:
     def idbm(self):
         """ 用生意经日数据制作商品 id 和编码对照表 """
         data_values = self.download.columns_to_list(
-            db_name='生意经2',
+            db_name='生意经3',
             table_name='宝贝指标',
             columns_name=['宝贝id', '商家编码', '行业类目'],
         )

mdbq/clean/clean_upload.py

@@ -487,22 +487,22 @@ class DataClean:
         report_names = [
             {
                 '文件简称': 'baobei',
-                '数据库名': '生意经2',
+                '数据库名': '生意经3',
                 '集合名称': '宝贝指标',
             },
             {
                 '文件简称': 'order',
-                '数据库名': '生意经2',
+                '数据库名': '生意经3',
                 '集合名称': '订单数据',
             },
             {
                 '文件简称': '省份城市分析',
-                '数据库名': '生意经2',
+                '数据库名': '生意经3',
                 '集合名称': '省份城市分析',
             },
             {
                 '文件简称': '店铺销售指标',
-                '数据库名': '生意经2',
+                '数据库名': '生意经3',
                 '集合名称': '店铺销售指标',
             },
         ]
@@ -538,25 +538,23 @@ class DataClean:
                 if name.endswith('.csv') and 'baobei' in name:
                     encoding = self.get_encoding(file_path=os.path.join(root, name))
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    pattern = re.findall(r'-(\d{4})(\d{2})(\d{2})\W', name)[0]
-                    df['日期'] = '-'.join(pattern)
+                    p = df.pop('日期')
+                    df.insert(loc=0, column='日期', value=p)
+                    df['日期'] = df['日期'].apply(lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', str(x))[0]) if int(x) > 0 else '')
                     df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    new_name = f'py_xg_天猫_baobeitrains_{'-'.join(pattern)}.csv'
+                    new_name = f'py_xg_{name}'
                     self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.csv') and 'order' in name:
-                    """ 这里不能使用表格原先的 gb2312， 会报错 """
-                    # encoding = self.get_encoding(file_path=os.path.join(root, name))
-                    df = pd.read_csv(os.path.join(root, name), encoding='gb18030', header=0, na_filter=False)
-                    pattern = re.findall(r'(.*)(\d{4})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})', name)[0]
-                    date1 ='-'.join(pattern[1:4])
-                    date2 = '-'.join(pattern[4:7])
-                    df.insert(loc=0, column='日期', value=date1)
-                    df.insert(loc=1, column='数据周期', value=f'{date1}_{date2}')
+                    """ 如果是手动下载的表格，这里不能使用表格原先的 gb2312， 会报错 """
+                    # df = pd.read_csv(os.path.join(root, name), encoding='gb18030', header=0, na_filter=False)
+                    encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
                     df.rename(columns={'宝贝标题': '商品标题', '宝贝链接': '商品链接'}, inplace=True)
-                    df['颜色编码'] = df['商家编码'].apply(
-                        lambda x: ''.join(re.findall(r' .*(\d{4})$', str(x))) if x else x)
-                    new_name = f'py_xg_天猫_order_{date1}_{date2}.csv'
+                    df['日期'] = df['日期'].apply(lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', str(x))[0]) if int(x) > 0 else '')
+                    df['商品id'] = df.apply(lambda x: re.sub(r'.*id=', '', x['商品链接']), axis=1)
+                    df = df[df['订单号'] != '']
+                    new_name = f'py_xg_{name}'
                     self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.csv') and '省份城市分析' in name:
@@ -585,27 +583,15 @@ class DataClean:
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.csv') and '店铺销售指标' in name:
                     # 生意经, 店铺指标，仅限月数据，实际日指标也可以
-                    name_st = re.findall(r'([\u4e00-\u9fa5]+)\(分日', name)
-                    if not name_st:
-                        print(f'{name}  正则提取文件名失败')
-                        os.remove(os.path.join(root, name))
-                        continue
                     encoding = self.get_encoding(file_path=os.path.join(root, name))
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
                         os.remove(os.path.join(root, name))
                         continue
-                    df['日期'] = df['日期'].astype(str).apply(
-                        lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', x)[0]) if x else x)
-                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
-                    # min_clm = str(df.min()['日期']).split(' ')[0]
-                    # max_clm = str(df.max()['日期']).split(' ')[0]
-                    min_clm = str(df['日期'].min()).split(' ')[0]
-                    max_clm = str(df['日期'].max()).split(' ')[0]
-                    new_name = f'py_xg_天猫_{name_st[0]}-{min_clm}_{max_clm}.csv'  # 保存时将(分日)去掉
+                    df['日期'] = df['日期'].apply(lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', str(x))[0]) if int(x) > 0 else '')
                     df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
+                    new_name = f'py_xg_{name}'
                     self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
                     os.remove(os.path.join(root, name))
 
@@ -832,6 +818,8 @@ class DataClean:
                     # df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     new_name = f'py_xg_{name}'
+                    if os.path.isfile(os.path.join(root, new_name)):
+                        os.remove(os.path.join(root, new_name))
                     os.rename(os.path.join(root, name), os.path.join(root, new_name))
                 elif name.endswith('.xlsx') and '京东商智_sku_商品明细' in name:
                     df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
@@ -950,6 +938,8 @@ class DataClean:
                 elif name.endswith('.csv') and ('商品类目属性' in name or '商品主图视频' in name or '商品sku属性' in name):
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    if os.path.isfile(os.path.join(root, new_name)):
+                        os.remove(os.path.join(root, new_name))
                     os.rename(os.path.join(root, name), os.path.join(root, new_name))
 
                 # 将数据传入 self.datas 等待更新进数据库
@@ -1096,29 +1086,17 @@ class DataClean:
                 if 'py_xg' not in name:  # 排除非目标文件
                     continue
 
-                if '天猫' in name and name.endswith('.csv') and 'baobei' in name:
-                    t_path = os.path.join(self.source_path, '天猫_生意经', '宝贝指标')
-                    bib(t_path, _as_month=True)
-                elif '天猫' in name and name.endswith('.csv') and '省份城市分析' in name:
-                    t_path = os.path.join(self.source_path, '天猫_生意经', '省份城市分析')
-                    bib(t_path, _as_month=True)
-                elif '天猫' in name and name.endswith('.csv') and '店铺销售指标' in name:
-                    t_path = os.path.join(self.source_path, '天猫_生意经', '店铺销售指标')
-                    bib(t_path, _as_month=False)
-                elif '天猫' in name and name.endswith('.csv') and 'order' in name:
-                    t_path = os.path.join(self.source_path, '天猫_生意经', '订单数据')
-                    bib(t_path, _as_month=False)
-                elif '淘宝' in name or '企业店' in name and name.endswith('.csv') and 'baobei' in name:
-                    t_path = os.path.join(self.source_path, '淘宝_生意经', '宝贝指标')
+                if name.endswith('.csv') and 'baobei' in name:
+                    t_path = os.path.join(self.source_path, '生意经', '宝贝指标')
                     bib(t_path, _as_month=True)
-                elif '淘宝' in name or '企业店' in name and name.endswith('.csv') and '省份城市分析' in name:
-                    t_path = os.path.join(self.source_path, '淘宝_生意经', '省份城市分析')
+                elif name.endswith('.csv') and '省份城市分析' in name:
+                    t_path = os.path.join(self.source_path, '生意经', '省份城市分析')
                     bib(t_path, _as_month=True)
-                elif '淘宝' in name or '企业店' in name and name.endswith('.csv') and '店铺销售指标' in name:
-                    t_path = os.path.join(self.source_path, '淘宝_生意经', '店铺销售指标')
+                elif name.endswith('.csv') and '店铺销售指标' in name:
+                    t_path = os.path.join(self.source_path, '生意经', '店铺销售指标')
                     bib(t_path, _as_month=False)
-                elif '淘宝' in name or '企业店' in name and name.endswith('.csv') and 'order' in name:
-                    t_path = os.path.join(self.source_path, '淘宝_生意经', '订单数据')
+                elif name.endswith('.csv') and 'order' in name:
+                    t_path = os.path.join(self.source_path, '生意经', '订单数据')
                     bib(t_path, _as_month=False)
 
     # @try_except
@@ -1585,7 +1563,7 @@ def main(is_mysql=False, is_company=False):
     # 清理聚合数据, mongodb 中没有聚合数据，所以只需要清理 mysql 即可
     optimize_data.op_data(
         db_name_lists=['聚合数据'],
-        days=3650,
+        days=100,
         is_mongo=False,
         is_mysql=True,
     )

mdbq/company/copysh.py

@@ -324,7 +324,7 @@ def op_data(days: int =100):
                 '推广数据_淘宝店',
                 '爱库存2',
                 '生意参谋3',
-                '生意经2',
+                '生意经3',
                 # '聚合数据',
                 '达摩盘3',
             ],
@@ -407,6 +407,8 @@ def main():
                 )
                 # print(conf)
                 myconfig.write_back(datas=conf)  # 写回文件生效
+            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
+            print(f'{now}数据完成！')
 
         t.sleep_minutes = 5  # 同步前休眠时间
         if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':

{mdbq-2.8.3.dist-info → mdbq-2.8.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.8.3
+Version: 2.8.5
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.8.3.dist-info → mdbq-2.8.5.dist-info}/RECORD

@@ -1,18 +1,18 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=FsKrOTCgwUgIuTKNf6rJSrV1g58al9qZ-yLTokWU_YY,72134
+mdbq/aggregation/aggregation.py,sha256=fLsYQO7LTUtVFaVOxXry4OZviiIpZMcTLplE-HXW9XY,71964
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
-mdbq/aggregation/mysql_types.py,sha256=2nMEeGGJXdxC3kp0xz2DJ3q6-1rfxi3GkRgDnsKyuNI,10934
+mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=79uwiM2WqNNFxGpE2wKz742PRq-ZGgFjdOV0vgptHdY,3513
-mdbq/aggregation/query_data.py,sha256=sLjCO5yDhGdVouA8x42sX5rugT2w9itYKanpk3iywCc,100065
+mdbq/aggregation/query_data.py,sha256=kSIXx4xhthCSaWKdQP4Za9AW-Iu98UUXg9SSWZ8ixO4,100065
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=rNiv_7wcOgYfks1wK2sLHWNyi7jProCZgHZSGjC7IlQ,83318
+mdbq/clean/clean_upload.py,sha256=LRXcvsLqON5NJ5rj7RbeV0750N2Jrjtmr4J7sNPNPoM,81544
 mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=tWSD7ZnPxSFrguKzI837glg90xoaxK4JL0IavtnMyGM,19943
+mdbq/company/copysh.py,sha256=gWaNgRe_rBHBBphch2AHkGAljZRdfqYBn_FC3m_l11A,20061
 mdbq/company/copysh_bak.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
 mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -45,7 +45,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=jHrdGWBJQaSywx7V-U4YuM6vWkwC5SR5tTOOdB3YU_c,17306
-mdbq-2.8.3.dist-info/METADATA,sha256=URhLHhurWFz89hBRcw66jx2ebsU23fIS7peIyMR_Iv0,243
-mdbq-2.8.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.8.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.8.3.dist-info/RECORD,,
+mdbq-2.8.5.dist-info/METADATA,sha256=YeGV1kaOhz9_4VCvMcBraZ0hMCk_tpdL3lJZP-2kda0,243
+mdbq-2.8.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.8.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.8.5.dist-info/RECORD,,