mdbq 2.6.8__py3-none-any.whl → 2.7.0__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1118,7 +1118,7 @@ class DatabaseUpdate:
 def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}, json_path=None, target_service='company'):
     """ 上传一个文件夹到 mysql 或者 mongodb 数据库 """
     if not os.path.isdir(path):
-        print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload_dir: 函数只接受文件夹路径，不是一个文件夹: {path}')
+        print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload_dir: 路径不存在或错误: {path}')
         return
 
     if dbs['mongodb']:
@@ -1326,10 +1326,10 @@ if __name__ == '__main__':
     # )
 
     # 上传一个目录到指定数据库
-    db_name = '生意经2'
-    table_name = '省份城市分析'
+    db_name = '推广数据2'
+    table_name = '营销场景报表'
     upload_dir(
-        path='/Users/xigua/数据中心/原始文件3/天猫_生意经/省份城市分析',
+        path='/Users/xigua/数据中心/原始文件3/天猫推广报表/营销场景报表',
         db_name=db_name,
         collection_name=table_name,
         dbs={'mysql': True, 'mongodb': False},

mdbq/aggregation/query_data.py

@@ -2316,7 +2316,7 @@ def main():
 if __name__ == '__main__':
     data_aggregation(
         service_databases=[{'company': 'mysql'}],
-        months=1,
-        is_juhe=False,  # 立即启动对聚合数据的清理工作
-        # less_dict=['生意参谋_直播场次分析'],  # 单独聚合某一个数据库
+        months=12,
+        is_juhe=True,  # 立即启动对聚合数据的清理工作
+        # less_dict=['营销场景报表'],  # 单独聚合某一个数据库
     )

mdbq/clean/clean_upload.py

@@ -468,7 +468,7 @@ class DataClean:
             {
                 '文件简称': 'order',
                 '数据库名': '生意经2',
-                '集合名称': '订单指标',
+                '集合名称': '订单数据',
             },
             {
                 '文件简称': '省份城市分析',
@@ -609,7 +609,7 @@ class DataClean:
             {
                 '文件简称': 'order',
                 '数据库名': '淘宝_生意经3',
-                '集合名称': '订单指标',
+                '集合名称': '订单数据',
             },
             {
                 '文件简称': '省份城市分析',
@@ -1587,7 +1587,7 @@ def main(service_databases=None, is_mysql=False):
     cn.dmp_tm(is_except=['except'])  # 达摩盘
     cn.tg_reports(is_except=['except'])  # 推广报表，天猫淘宝共同清洗
     cn.syj_reports_tm(is_except=['except'])  # 天猫生意经
-    # # 淘宝生意经，不可以和天猫同时运行
+    """ 淘宝生意经，不可以和天猫同时运行 """
     # cn.syj_reports_tb(is_except=['except'])  # 淘宝生意经，不可以和天猫同时运行
     cn.jd_reports(is_except=['except'])  # 清洗京东报表
     cn.sp_scene_clean(is_except=['except'])  # 商品素材

mdbq/company/copysh.py

@@ -321,7 +321,7 @@ def op_data(days: int =100):
         # 清理所有非聚合数据的库
         optimize_data.op_data(
             db_name_lists=[
-                '京东数据2',
+                '京东数据3',
                 '属性设置3',
                 '推广数据2',
                 '推广数据_淘宝店',
@@ -367,6 +367,7 @@ def main():
             op_data(days=100)
 
         t.sleep_minutes = 5  # 同步前休眠时间
+        # 4. 同步共享文件
         t.tb_file()
         time.sleep(600)  # 检测间隔
 

mdbq/mysql/recheck_mysql.py

@@ -0,0 +1,201 @@
+# -*- coding: UTF-8 –*-
+import os
+import time
+import pandas as pd
+import warnings
+import datetime
+from dateutil.relativedelta import relativedelta
+from mdbq.config import get_myconf
+from mdbq.mysql import mysql
+from mdbq.mysql import s_query
+
+""" 这是一个临时文件， 用来修改原始文件中大量 csv 文件中的场景名字（万相台报表） """
+warnings.filterwarnings('ignore')
+
+
+def id_account_rpt(id_rpt):
+    """
+    id_rpt: 传入原二级场景id/原二级场景名字 ，返回其他键值
+    只可以旧推新，不可以新推旧
+    例如： 粉丝快 -> 人群推广， 精准人群推广 -> 人群推广
+    但不可以：人群推广 对应着旧的多个值，会发生问题
+    """
+    if '="' in str(id_rpt):
+        id_rpt = str(id_rpt).replace('="', '')
+        id_rpt = str(id_rpt).replace('"', '')
+    _id_account_rpt = [
+        {
+            '原二级场景id': '436',
+            '原二级场景名字': '全站推广',
+            '场景id': '436',
+            '场景名字': '全站推广',
+        },
+        {
+            '原二级场景id': '407',
+            '原二级场景名字': '粉丝快',
+            '场景id': '372',
+            '场景名字': '人群推广',
+        },
+        {
+            '原二级场景id': '114',
+            '原二级场景名字': '货品加速',
+            '场景id': '376',
+            '场景名字': '货品运营',
+        },
+        {
+            '原二级场景id': '372',
+            '原二级场景名字': '精准人群推广',
+            '场景id': '372',
+            '场景名字': '人群推广',
+        },
+        {
+            '原二级场景id': '371',
+            '原二级场景名字': '关键词推广',
+            '场景id': '371',
+            '场景名字': '关键词推广',
+        },
+        {
+            '原二级场景id': '361',
+            '原二级场景名字': '全店智投',
+            '场景id': '361',
+            '场景名字': '全店智投',
+        },
+        {
+            '原二级场景id': '183',
+            '原二级场景名字': '超级短视频',
+            '场景id': '183',
+            '场景名字': '超级短视频',
+        },
+        {
+            '原二级场景id': '154',
+            '原二级场景名字': '活动加速',
+            '场景id': '154',
+            '场景名字': '活动加速',
+        },
+        {
+            '原二级场景id': '133',
+            '原二级场景名字': '会员快',
+            '场景id': '372',
+            '场景名字': '人群推广',
+        },
+        {
+            '原二级场景id': '108',
+            '原二级场景名字': '超级直播',
+            '场景id': '108',
+            '场景名字': '超级直播',
+        },
+        {
+            '原二级场景id': '105',
+            '原二级场景名字': '上新快',
+            '场景id': '105',
+            '场景名字': '上新快',
+        },
+        {
+            '原二级场景id': '78',
+            '原二级场景名字': '拉新快',
+            '场景id': '372',
+            '场景名字': '人群推广',
+        },
+    ]
+
+    for data in _id_account_rpt:
+        if id_rpt == data['原二级场景id'] or id_rpt == data['原二级场景名字']:
+            return data
+
+
+class ReCheckMysql:
+    def __init__(self, target_service):
+        username, password, host, port = get_myconf.select_config_values(target_service=target_service,
+                                                                         database='mysql')
+        self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+        self.months = 1  # 读取近 num 个月的数据, 0 表示读取当月的数据
+
+    @staticmethod
+    def months_data(num=0, end_date=None):
+        """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
+        if not end_date:
+            end_date = datetime.datetime.now()
+        start_date = end_date - relativedelta(months=num)  # n 月以前的今天
+        start_date = f'{start_date.year}-{start_date.month}-01'  # 替换为 n 月以前的第一天
+        return pd.to_datetime(start_date), pd.to_datetime(end_date)
+
+    def recheck_cols(self, db_name, table_name, service_name='company'):
+        start_date, end_date = self.months_data(num=self.months)
+        df = self.download.data_to_df(
+            db_name=db_name,
+            table_name=table_name,
+            start_date=start_date,
+            end_date=end_date,
+            projection={},
+        )
+        # df.to_csv('/Users/xigua/Downloads/test_before.csv', index=False, header=True, encoding='utf-8_sig')
+        # 调用 self.id_account_rpt 函数，根据场景id 修改对应的场景名字，如果没有匹配则不修改
+        df['场景名字'] = df.apply(lambda x: id_account_rpt(x['场景id']) if id_account_rpt(x['场景id']) else x['场景名字'], axis=1)
+        # df.to_csv('/Users/xigua/Downloads/test.csv', index=False, header=True, encoding='utf-8_sig')
+
+        username, password, host, port = get_myconf.select_config_values(
+            target_service=service_name,
+            database='mysql',
+        )
+        m = mysql.MysqlUpload(
+            username=username,
+            password=password,
+            host=host,
+            port=port,
+        )
+        m.df_to_mysql(
+            df=df,
+            db_name=db_name,
+            table_name=table_name,
+            move_insert=True,  # 先删除，再插入
+            df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+            drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+            filename='',  # 用来追踪处理进度
+            service_database={'company': 'mysql'},  # 字典
+        )
+
+
+def recheck_csv():
+    path = '/Users/xigua/数据中心/原始文件2/推广报表34324234'
+    for root, dirs, files in os.walk(path, topdown=False):
+        for name in files:
+            if '~' in name or 'baidu' in name or 'Ds_' in name or 'xunlei' in name:
+                continue
+            if name.endswith('.csv'):
+                df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                if '场景ID' not in df.columns.tolist():
+                    continue
+                if '原二级场景名字' in df.columns.tolist() and '原二级场景ID' in df.columns.tolist():
+                    df['原二级场景ID'].replace(to_replace='="', value='', regex=True, inplace=True)
+                    df['原二级场景ID'].replace(to_replace='"', value='', regex=True, inplace=True)
+                if '场景名字' in df.columns.tolist() and '场景ID' in df.columns.tolist():
+                    df['场景ID'].replace(to_replace='="', value='', regex=True, inplace=True)
+                    df['场景ID'].replace(to_replace='"', value='', regex=True, inplace=True)
+                if '场景名字' in df.columns.tolist() and '场景ID' in df.columns.tolist() and '原二级场景名字' not in df.columns.tolist():
+                    df.rename(columns={
+                        '场景名字': '原二级场景名字',
+                        '场景ID': '原二级场景ID',
+                    }, inplace=True)
+                    # 根据 id 修正 场景名字
+                    df['原二级场景名字'] = df.apply(
+                        lambda x: id_account_rpt(x['原二级场景ID'])['原二级场景名字'] if id_account_rpt(x['原二级场景ID']) else x['原二级场景名字'], axis=1)
+                    # 根据原场景id获取新场景名字
+                    df['场景名字'] = df.apply(
+                        lambda x: id_account_rpt(x['原二级场景ID'])['场景名字'] if id_account_rpt(x['原二级场景ID']) else '', axis=1)
+                    # 根据原场景id获取新场景id
+                    df['场景ID'] = df.apply(
+                        lambda x: id_account_rpt(x['原二级场景ID'])['场景id'] if id_account_rpt(x['原二级场景ID']) else '', axis=1)
+                print(name)
+                df.to_csv(os.path.join(root, name), index=False, header=True, encoding='utf-8_sig')
+
+
+if __name__ == '__main__':
+    # r = ReCheckMysql(target_service='company')
+    # r.months = 100
+    # r.recheck_cols(
+    #     db_name='推广数据2',
+    #     table_name='营销场景报表',
+    #     service_name='company',
+    # )
+
+    recheck_csv()

{mdbq-2.6.8.dist-info → mdbq-2.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.6.8
+Version: 2.7.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.6.8.dist-info → mdbq-2.7.0.dist-info}/RECORD

@@ -1,18 +1,18 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=fnXBRxATlaCohx_dzAIewVlPI0d8L-2QY6wth9ENCwA,76594
+mdbq/aggregation/aggregation.py,sha256=v_5VM-InaDDvDNjAy_b8xsc38-vf78GkqoXjoe6MZ8U,76569
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=gdScrgTAb6RbXHZy1LitX7lggMGn1GTLhkYSgztfwew,4903
-mdbq/aggregation/query_data.py,sha256=m7Y2xSazPYKvy51yPK6n_Izsv5cjV83oHsiNc7N4fyA,102779
+mdbq/aggregation/query_data.py,sha256=gqvKDgg3jrfCcI1VudrnQLJgKHUThZVTqS1zOQ5wgMk,102766
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=4DNoSQBUYyn6OsdAP4WJoqWneReeHlvmctXyS5dQvIU,86640
+mdbq/clean/clean_upload.py,sha256=X5WcWm7kkGZDMpk8p0vMq-SFIcrSL1DmVCYWbxYmLVI,86644
 mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=sisL5eo3D5HGGYvRw46xGqnqFaI3SxfBnoa-Y7zknus,17541
+mdbq/company/copysh.py,sha256=3ZYm_rTE8nXcbgZlyHsa0y-RAkZ8vcmwkxMy_Jj4F2k,17574
 mdbq/company/copysh_bak.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
 mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -28,6 +28,7 @@ mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/mysql.py,sha256=9IIyKYU81SXglz6GqVTz0-kCE2dhFuwpQAhUym-yjuc,47135
+mdbq/mysql/recheck_mysql.py,sha256=jHQSlQy0PlQ_EYICQv_2nairUX3t6OIwPtSELKIpjkY,8702
 mdbq/mysql/s_query.py,sha256=37GGHzRpycfUjsYEoQgDpdEs9JwjW-LxFXnGwwP2b2Q,8403
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -43,7 +44,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=4Y5zd64hZUFtll8AdpUc2napDas-La-A6XzAhb2mLv0,17157
-mdbq-2.6.8.dist-info/METADATA,sha256=0f19adUwFUrRTcAT5TvVboqz7L0X2CKv0x5acWdpIYw,245
-mdbq-2.6.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.6.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.6.8.dist-info/RECORD,,
+mdbq-2.7.0.dist-info/METADATA,sha256=i_0WznHsXfCR0sToIhC5S4mv3hv1qMUOFFdws6FocOg,245
+mdbq-2.7.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.7.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.7.0.dist-info/RECORD,,