mdbq 1.9.1__py3-none-any.whl → 1.9.3__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -61,7 +61,6 @@ class DatabaseUpdate:
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.ini' in name or '$RECYCLE.BIN' in name or 'Icon' in name:
                     continue
-
                 db_name = None  # 初始化/重置变量，避免进入下一个循环
                 collection_name = None
                 for data in datas:  # 根据标题对照表适配 db_name 和 collection_name
@@ -189,6 +188,22 @@ class DatabaseUpdate:
                             collection_name='店铺来源_日数据_旧版'
                 elif name.endswith('.csv') and '客户运营平台_客户列表' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                elif name.endswith('.xlsx') and '直播分场次效果' in name:
+                    pattern = re.findall(r'(\d{4}-\d{2}-\d{2})_(\d{4}-\d{2}-\d{2})', name)
+                    if pattern:
+                        continue
+                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    if len(df) == 0:
+                        print(f'{name} 报表数据为空')
+                        continue
+                    df.replace(to_replace=['--'], value='', regex=False, inplace=True)
+                    df.replace(to_replace=[','], value='', regex=True, inplace=True)
+                    df['直播开播时间'] = pd.to_datetime(df['直播开播时间'], format='%Y-%m-%d %H:%M:%S', errors='ignore')
+                    df.insert(loc=0, column='日期', value=df['直播开播时间'])
+                    df['日期'] = df['日期'].apply(
+                        lambda x: pd.to_datetime(str(x).split(' ')[0], format='%Y-%m-%d', errors='ignore') if x else x)
+                    df.insert(loc=1, column='店铺', value='万里马官方旗舰店')
+
                 elif name.endswith('.xls') and '生意参谋' in name and '无线店铺三级流量来源详情' in name:
                     # 店铺来源，手淘搜索，关键词
                     pattern = re.findall(r'(\d{4}-\d{2}-\d{2})_(\d{4}-\d{2}-\d{2})', name)
@@ -459,10 +474,20 @@ class DatabaseUpdate:
                 elif name.endswith('.xlsx') and '搜索分析-排名定位-商品词下排名' in name:
                     # 京东商品词下排名
                     try:
+                        pattern = re.findall(r'(\d{4}-\d{2}-\d{2})-(\d{4}-\d{2}-\d{2})', name)
+                        if not pattern:
+                            continue
+                        if pattern[0][0] != pattern[0][1]:
+                            print(f'{name}: 检测到数据周期异常，仅支持7天数据')
+                            continue
                         df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
                         if len(df) == 0:
                             print(f'{name} 报表数据为空')
                             continue
+                        if len(df.columns.tolist()) < 20:
+                            print(f'{name}: 报表可能缺失诊断数据')
+                            os.remove(os.path.join(root, name))
+                            continue
                         df.rename(columns={'商品的ID': 'skuid'}, inplace=True)
                         for col in ['词人气', '搜索点击率']:
                             if col in df.columns.tolist():
@@ -1115,12 +1140,23 @@ if __name__ == '__main__':
     #     database='mysql'
     # )
 
-    db_name = '生意经2'
-    table_name = '省份城市分析'
-    upload_dir(
-        path='/Users/xigua/数据中心/原始文件2/生意经/地域分布',
-        db_name=db_name,
-        collection_name=table_name,
-        dbs={'mysql': True, 'mongodb': False},
-    )
+    # db_name = '生意经2'
+    # table_name = '省份城市分析'
+    # upload_dir(
+    #     path='/Users/xigua/数据中心/原始文件2/生意经/地域分布',
+    #     db_name=db_name,
+    #     collection_name=table_name,
+    #     dbs={'mysql': True, 'mongodb': False},
+    # )
+    #
 
+    # 新版 数据分类
+    dp = DatabaseUpdate(path='/Users/xigua/Downloads')
+    dp.new_unzip(is_move=True)
+    dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
+    # 将 self.datas 更新至数据库
+    dp.upload_df(service_databases=[
+        # {'home_lx': 'mongodb'},
+        {'company': 'mysql'},
+        # {'nas': 'mysql'},
+    ])

mdbq/aggregation/query_data.py

@@ -454,6 +454,50 @@ class MysqlDatasQuery:
         )
         return df
 
+    def zb_ccfx(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '店铺': 1,
+            '场次信息': 1,
+            '场次id': 1,
+            '直播开播时间': 1,
+            '开播时长': 1,
+            '封面图点击率': 1,
+            '观看人数': 1,
+            '观看次数': 1,
+            '新增粉丝数': 1,
+            '流量券消耗': 1,
+            '观看总时长（秒）': 1,
+            '人均观看时长（秒）': 1,
+            '次均观看时长（秒）': 1,
+            '商品点击人数': 1,
+            '商品点击次数': 1,
+            '商品点击率': 1,
+            '加购人数': 1,
+            '加购件数': 1,
+            '加购次数': 1,
+            '成交金额（元）': 1,
+            '成交人数': 1,
+            '成交件数': 1,
+            '成交笔数': 1,
+            '成交转化率': 1,
+            '退款人数': 1,
+            '退款笔数': 1,
+            '退款件数': 1,
+            '退款金额（元）': 1,
+            '预售定金支付金额（元）': 1,
+            '预售预估总金额（元）': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='生意参谋2',
+            table_name='直播场次分析',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
+
 class GroupBy:
     """
     数据聚合和导出
@@ -1016,6 +1060,9 @@ class GroupBy:
                 }
             )
             return df
+        elif '直播场次分析' in table_name:
+            df.drop_duplicates(subset=['日期', '直播开播时间', '观看人数'], keep='first', inplace=True, ignore_index=True)
+            return df
         else:
             print(f'<{table_name}>: Groupby 类尚未配置，数据为空')
             return pd.DataFrame({})
@@ -1056,7 +1103,6 @@ class GroupBy:
         df['毛利率'] = df.apply(lambda x: round((x['销售额'] - x['商品成本']) / x['销售额'], 4) if x['销售额'] > 0 else 0, axis=1)
         df['盈亏'] = df.apply(lambda x: x['商品毛利'] - x['花费'], axis=1)
         return df
-
     def performance_concat(self, bb_tg=True):
         tg,  zb, pxb = self.data_tgyj['天猫汇总表调用'], self.data_tgyj['天猫_超级直播'], self.data_tgyj['天猫_品销宝账户报表']
         zb.rename(columns={
@@ -1385,6 +1431,12 @@ def data_aggregation(service_databases=[{}], months=1):
                     '唯一主键': ['日期', '关键词', '访客数'],
                     '数据主体': sdq.tm_search(),
                 },
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '生意参谋_直播场次分析',
+                    '唯一主键': ['日期', '直播开播时间'],
+                    '数据主体': sdq.zb_ccfx(),
+                },
             ]
             for items in data_dict:  # 遍历返回结果
                 db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']
@@ -1402,7 +1454,7 @@ def data_aggregation(service_databases=[{}], months=1):
                         service_database=service_database,
                     )
                     g.sp_index_datas = pd.DataFrame()  # 重置，不然下个循环会继续刷入数据库
-                # g.as_csv(df=df, filename=table_name + '.csv')  # 导出 csv
+                # # g.as_csv(df=df, filename=table_name + '.csv')  # 导出 csv
                 if '日期' in df.columns.tolist():
                     m.df_to_mysql(
                         df=df,
@@ -1483,7 +1535,7 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=24)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=1)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作
 

mdbq/clean/data_clean.py

@@ -443,7 +443,26 @@ class DataClean:
                         if self.set_up_to_mysql:
                             m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_自助取数_店铺流量_月数据')
                         os.remove(os.path.join(root, name))
-
+                    elif name.endswith('.xlsx') and '直播分场次效果' in name:
+                        pattern = re.findall(r'(\d{4}-\d{2}-\d{2})_(\d{4}-\d{2}-\d{2})', name)
+                        if pattern:
+                            continue
+                        df = pd.read_excel(os.path.join(root, name), header=0)
+                        if len(df) == 0:
+                            print(f'{name} 报表数据为空')
+                            continue
+                        df.replace(to_replace=['--'], value='0', regex=False, inplace=True)
+                        df.replace(to_replace=[','], value='', regex=True, inplace=True)
+                        df['直播开播时间'] = pd.to_datetime(df['直播开播时间'], format='%Y-%m-%d %H:%M:%S', errors='ignore')
+                        df.insert(loc=0, column='日期', value=df['直播开播时间'])
+                        df['日期'] = df['日期'].apply(lambda x: pd.to_datetime(str(x).split(' ')[0], format='%Y-%m-%d', errors='ignore') if x else x)
+                        df.insert(loc=1, column='店铺', value='万里马官方旗舰店')
+                        min_clm = str(df.min()['直播开播时间']).split(' ')[0]
+                        max_clm = str(df.max()['直播开播时间']).split(' ')[0]
+                        new_name = f'{os.path.splitext(name)[0]}_{min_clm}_{max_clm}.csv'
+                        new_name = re.sub(r' ?(\(\d+\))', '',new_name)
+                        self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
+                        os.remove(os.path.join(root, name))
                     elif name.endswith('.csv') and 'baobei' in name:
                         # 生意经宝贝指标日数据
                         # print(name)
@@ -808,6 +827,12 @@ class DataClean:
                         os.remove(os.path.join(root, name))
                     elif name.endswith('.xlsx') and '搜索分析-排名定位-商品词下排名' in name:
                         # 京东商品词下排名
+                        pattern = re.findall(r'(\d{4}-\d{2}-\d{2})-(\d{4}-\d{2}-\d{2})', name)
+                        if not pattern:
+                            continue
+                        if pattern[0][0] != pattern[0][1]:
+                            print(f'{name}: 检测到数据周期异常，仅支持7天数据')
+                            continue
                         new_name = os.path.splitext(name)[0] + '.csv'
                         # print(name)
                         df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
@@ -815,6 +840,10 @@ class DataClean:
                             print(f'{name} 报表数据为空')
                             os.remove(os.path.join(root, name))
                             continue
+                        if len(df.columns.tolist()) < 20:
+                            print(f'{name}: 报表可能缺失诊断数据')
+                            os.remove(os.path.join(root, name))
+                            continue
                         df.rename(columns={'商品的ID': 'skuid'}, inplace=True)
                         df['skuid'] = df['skuid'].apply(lambda x: f'="{x}"' if x and '=' not in str(x) else x)
                         self.save_to_csv(df, root, new_name)
@@ -1192,6 +1221,12 @@ class DataClean:
                 elif name.endswith('.csv') and '客户运营平台_客户列表' in name:
                     t_path = str(pathlib.Path(self.source_path, '生意参谋/客户运营平台'))
                     bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '直播分场次效果' in name:
+                    pattern = re.findall(r'(\d{4}-\d{2}-\d{2})_(\d{4}-\d{2}-\d{2})', name)
+                    if not pattern:
+                        continue
+                    t_path = str(pathlib.Path(self.source_path, '生意参谋/直播场次分析'))
+                    bib(t_path, _as_month=True)
                 #  京东分界线   ------- 开始标记
                 #  京东分界线
                 elif name.endswith('.csv') and '全部渠道_商品明细' in name:
@@ -1435,11 +1470,11 @@ def main():
     c.set_up_to_mysql = False
     c.new_unzip(is_move=True)  # 解压文件
     c.change_and_sort()
-    # c.move_all()  # 移到文件到原始文件夹
+    c.move_all()  # 移到文件到原始文件夹
     # c.attribute()  # 商品素材重命名和分类
 
 
 if __name__ == '__main__':
-    # main()
+    main()
     username, password, host, port = get_myconf.select_config_values(target_service='aliyun', database='mongodb')
     print(username, password, host, port)

mdbq/mysql/mysql.py

@@ -57,6 +57,18 @@ class MysqlUpload:
         }
         self.filename = None
 
+    @staticmethod
+    def try_except(func):  # 在类内部定义一个异常处理方法
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                print(f'{func.__name__}, {e}')  # 将异常信息返回
+
+        return wrapper
+
+    @try_except
     def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None):
         """
         将 df 写入数据库
@@ -81,6 +93,9 @@ class MysqlUpload:
         else:
             print(f'{db_name}: {table_name} 传入的 df 不是有效的 dataframe 结构, {self.filename}')
             return
+        if not db_name or db_name == 'None':
+            print(f'{db_name} 不能为 None')
+            return
 
         cv = converter.DataFrameConverter()
         df = cv.convert_df_cols(df=df)  # 清理 dataframe 非法值

{mdbq-1.9.1.dist-info → mdbq-1.9.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.9.1
+Version: 1.9.3
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.9.1.dist-info → mdbq-1.9.3.dist-info}/RECORD

@@ -1,15 +1,15 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=LEOpmH_Kyf08N4z9pdqMznRicLp9ON28UJZyZMZS1Uo,65029
+mdbq/aggregation/aggregation.py,sha256=A_Zy6FtlmrVmjjXaioUecRYVVDpKQHi3Ase-z8nSV14,67166
 mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
-mdbq/aggregation/query_data.py,sha256=u4f5Pdy_foD225FzNigOJ20FCiu0wedP2gVvlai-mUw,70447
+mdbq/aggregation/query_data.py,sha256=8Cc3fj4-lLkQyazOtb-_CePiiCog3omeiaJ577dfZXU,72435
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/data_clean.py,sha256=W8mqA-Ur9cXbZAL9ew0e24hwWqk2fvdUzhqbd7ZLtRg,98030
+mdbq/clean/data_clean.py,sha256=LrnliUyiI_9zwbI4GprDuYS-1XoZmSpI_DK6hn-vg9I,100661
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=4PGjvmPzvrmstOaAwHQGFXIGCWqqNXZEOYf1QdUvMlI,17762
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -24,7 +24,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=UKnBmywqTzc0VJfZGlC-9KzV7I--9P7H-jspUp_IvtU,44071
+mdbq/mysql/mysql.py,sha256=LJJja2S5OWc-3lOHDmsVFJieFM3U69pbyWBEYQVn7P4,44541
 mdbq/mysql/s_query.py,sha256=fIQvQKPyV7rvSUuxVWXv9S5FmCnIM4GHKconE1Zn5BA,8378
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.9.1.dist-info/METADATA,sha256=QIGSr1tYF_hYmqS_Nkn0b37ZMgge-ivGIbZSAJ7tSy4,245
-mdbq-1.9.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-1.9.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.9.1.dist-info/RECORD,,
+mdbq-1.9.3.dist-info/METADATA,sha256=gEFZjDAiQ8_FgTxoXwPOdv5UEMcL_so2cCd3yHr3W4A,245
+mdbq-1.9.3.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.9.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.9.3.dist-info/RECORD,,

{mdbq-1.9.1.dist-info → mdbq-1.9.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any