mdbq 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

mdbq/aggregation/mysql_types.py

@@ -154,90 +154,91 @@ class DataTypes:
             return {}, cl, None, None  # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
 
 
-def mysql_all_dtypes(db_name=None, table_name=None, path=None, system_name=None):
+def mysql_all_dtypes(db_name=None, table_name=None, service_databases=[{'home_lx': 'mysql'}], path=None, system_name=None):
     """
     更新笔记本 mysql 中所有数据库的 dtypes 信息到本地 json
     """
-
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
-    config = {
-        'host': host,
-        'port': port,
-        'user': username,
-        'password': password,
-        'charset': 'utf8mb4',  # utf8mb4 支持存储四字节的UTF-8字符集
-        'cursorclass': pymysql.cursors.DictCursor,
-    }
-
-    connection = pymysql.connect(**config)  # 连接数据库
-    with connection.cursor() as cursor:
-        sql = "SHOW DATABASES;"
-        cursor.execute(sql)
-        db_name_lists = cursor.fetchall()
-        db_name_lists = [item['Database'] for item in db_name_lists]
-        connection.close()
-
-    sys_lists = ['information_schema', 'mysql', 'performance_schema', 'sakila', 'sys']
-    db_name_lists = [item for item in db_name_lists if item not in sys_lists]
-
-    # db_name_lists = [
-    #     '京东数据2',
-    #     '推广数据2',
-    #     '市场数据2',
-    #     '生意参谋2',
-    #     '生意经2',
-    #     '属性设置2',
-    #     '聚合数据',
-    # ]
-    results = []  # 返回结果示例: [{'云电影': '电影更新'}, {'生意经2': 'e3_零售明细统计'}]
-    for db_ in db_name_lists:
-        config.update({'database': db_})  # 添加更新 config 字段
-        connection = pymysql.connect(**config)  # 连接数据库
-        try:
+    for service_database in service_databases:
+        for service_name, database in service_database.items():
+            username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+            config = {
+                'host': host,
+                'port': port,
+                'user': username,
+                'password': password,
+                'charset': 'utf8mb4',  # utf8mb4 支持存储四字节的UTF-8字符集
+                'cursorclass': pymysql.cursors.DictCursor,
+            }
+
+            connection = pymysql.connect(**config)  # 连接数据库
             with connection.cursor() as cursor:
-                sql = f"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{db_}';"
-                sql = "SHOW TABLES;"
+                sql = "SHOW DATABASES;"
                 cursor.execute(sql)
-                res_tables = cursor.fetchall()
-                for res_table in res_tables:
-                    for k, v in res_table.items():
-                        results.append({db_: v})
-        except:
-            pass
-        finally:
-            connection.close()
-        time.sleep(0.5)
-
-    d = DataTypes(path=path, system_name=system_name)
-    for result in results:
-        for db_n, table_n in result.items():
-            # print(db_n, table_n, db_name, table_name)
-            if db_name and table_name:  # 下载一个指定的数据表
-                if db_name != db_n or table_name != table_n:
-                    continue
-            elif db_name:  # 下载一个数据库的所有数据表
-                if db_name != db_n:
-                    continue
-            # 如果 db_name 和 table_name 都不指定，则下载所有数据库的所有数据表
-            print(f'获取列信息 数据库: < {db_n} >, 数据表: < {table_n} >')
-            sq = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-            # 获取数据表的指定列, 返回列表
-            # [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
-            name_type = sq.dtypes_to_list(db_name=db_n, table_name=table_n)
-            if name_type:
-                dtypes = {item['COLUMN_NAME']: item['COLUMN_TYPE'] for item in name_type}
-                dtypes = {'mysql': {db_n: {table_n: dtypes}}}
-                d.get_mysql_types(
-                    dtypes=dtypes, 
-                    cl='mysql', 
-                    db_name=db_n,
-                    table_name=table_n,
-                    is_file_dtype=True  # True表示旧文件有限
-                )
-            else:
-                print(f'数据库回传数据(name_type)为空')
-    # print(d.datas)
-    d.as_json_file()
+                db_name_lists = cursor.fetchall()
+                db_name_lists = [item['Database'] for item in db_name_lists]
+                connection.close()
+
+            sys_lists = ['information_schema', 'mysql', 'performance_schema', 'sakila', 'sys']
+            db_name_lists = [item for item in db_name_lists if item not in sys_lists]
+
+            # db_name_lists = [
+            #     '京东数据2',
+            #     '推广数据2',
+            #     '市场数据2',
+            #     '生意参谋2',
+            #     '生意经2',
+            #     '属性设置2',
+            #     '聚合数据',
+            # ]
+            results = []  # 返回结果示例: [{'云电影': '电影更新'}, {'生意经2': 'e3_零售明细统计'}]
+            for db_ in db_name_lists:
+                config.update({'database': db_})  # 添加更新 config 字段
+                connection = pymysql.connect(**config)  # 连接数据库
+                try:
+                    with connection.cursor() as cursor:
+                        sql = f"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{db_}';"
+                        sql = "SHOW TABLES;"
+                        cursor.execute(sql)
+                        res_tables = cursor.fetchall()
+                        for res_table in res_tables:
+                            for k, v in res_table.items():
+                                results.append({db_: v})
+                except:
+                    pass
+                finally:
+                    connection.close()
+                time.sleep(0.5)
+
+            d = DataTypes(path=path, system_name=system_name)
+            for result in results:
+                for db_n, table_n in result.items():
+                    # print(db_n, table_n, db_name, table_name)
+                    if db_name and table_name:  # 下载一个指定的数据表
+                        if db_name != db_n or table_name != table_n:
+                            continue
+                    elif db_name:  # 下载一个数据库的所有数据表
+                        if db_name != db_n:
+                            continue
+                    # 如果 db_name 和 table_name 都不指定，则下载所有数据库的所有数据表
+                    print(f'获取列信息 数据库: < {db_n} >, 数据表: < {table_n} >')
+                    sq = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+                    # 获取数据表的指定列, 返回列表
+                    # [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
+                    name_type = sq.dtypes_to_list(db_name=db_n, table_name=table_n)
+                    if name_type:
+                        dtypes = {item['COLUMN_NAME']: item['COLUMN_TYPE'] for item in name_type}
+                        dtypes = {'mysql': {db_n: {table_n: dtypes}}}
+                        d.get_mysql_types(
+                            dtypes=dtypes,
+                            cl='mysql',
+                            db_name=db_n,
+                            table_name=table_n,
+                            is_file_dtype=True  # True表示旧文件有限
+                        )
+                    else:
+                        print(f'数据库回传数据(name_type)为空')
+            # print(d.datas)
+            d.as_json_file()
 
 
 if __name__ == '__main__':

mdbq/aggregation/query_data.py

@@ -494,7 +494,45 @@ class GroupBy:
                 print(f'{filename}: sort_values排序参数错误！')
         df.to_excel(os.path.join(path, filename + '.xlsx'), index=index, header=header, engine=engine, freeze_panes=freeze_panes)
         
-        
+
+def data_aggregation_one(service_databases=[{}], months=1, system_name=None,):
+    """
+    # 单独处理某一个聚合数据库，修改添加 data_dict 的值
+    """
+    for service_database in service_databases:
+        for service_name, database in service_database.items():
+            sdq = MysqlDatasQuery(target_service=service_name)  # 实例化数据处理类
+            sdq.months = months  # 设置数据周期， 1 表示近 2 个月
+            g = GroupBy()  # 实例化数据聚合类
+            # 实例化数据库连接
+            username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+            m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+
+            # 从数据库中获取数据, 返回包含 df 数据的字典
+            # 单独处理某一个聚合数据库，在这里修改添加 data_dict 的值
+            data_dict = [
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '天猫_店铺来源_日数据_旧版',
+                    '唯一主键': ['日期', '一级来源', '二级来源', '三级来源'],
+                    '数据主体': sdq.dplyd(),
+                },
+            ]
+            for items in data_dict:  # 遍历返回结果
+                db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']
+                df = g.groupby(df=df, table_name=table_name, is_maximize=True)  # 2. 聚合数据
+                # g.as_csv(df=df, filename=table_name + '.csv')  # 导出 csv
+                m.df_to_mysql(
+                    df=df,
+                    db_name=db_name,
+                    table_name=table_name,
+                    drop_dup=False,
+                    icm_update=unique_key_list,
+                    system_name=system_name,
+                    service_databases=service_databases,
+                )  # 3. 回传数据库
+
+
 def data_aggregation(service_databases=[{}], months=1, system_name=None,):
     """
     1. 从数据库中读取数据
@@ -561,6 +599,7 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
                     drop_dup=False,
                     icm_update=unique_key_list,
                     system_name=system_name,
+                    service_databases=service_databases,
                 )  # 3. 回传数据库
             res = g.performance(bb_tg=True)   # 盈亏表，依赖其他表，单独做
             m.df_to_mysql(
@@ -570,6 +609,7 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
                 drop_dup=False,
                 icm_update=['日期', '商品id'],  # 设置唯一主键
                 system_name = system_name,
+                service_databases=service_databases,
             )
             res = g.performance(bb_tg=False)  # 盈亏表，依赖其他表，单独做
             m.df_to_mysql(
@@ -579,6 +619,7 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
                 drop_dup=False,
                 icm_update=['日期', '商品id'],  # 设置唯一主键
                 system_name=system_name,
+                service_databases=service_databases,
             )
 
     # 这里要注释掉，不然 copysh.py 可能有问题，这里主要修改配置文件，后续触发 home_lx 的 optimize_datas.py(有s)程序进行全局清理
@@ -586,5 +627,6 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=1, system_name='company')
+    # data_aggregation(service_databases=[{'company': 'mysql'}], months=1, system_name='company')
+    data_aggregation_one(service_databases=[{'company': 'mysql'}], months=10, system_name='company')
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

mdbq/company/copysh.py

@@ -289,7 +289,7 @@ class UpdateMysql:
             return False, self.d_path
 
 
-def op_data(days: int =3650):
+def op_data(days: int =100):
 
     # 清理数据库， 除了 聚合数据
     if socket.gethostname() == 'company':  # 公司台式机自身运行
@@ -322,11 +322,11 @@ def op_data(days: int =3650):
                 '属性设置2',
                 # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
             ],
-            days=3650,
+            days=days,
         )
 
         # 数据聚合
-        query_data.data_aggregation(service_databases=[{'home_lx': 'mysql'}], months=1, system_name='home_lx')
+        query_data.data_aggregation(service_databases=[{'home_lx': 'mysql'}], months=3, system_name='home_lx')
         time.sleep(60)
 
         # 清理聚合数据
@@ -352,7 +352,7 @@ def main():
             w = update_conf.UpdateConf()
             w.update_config(filename='.copysh_conf', option='ch_record', new_value='False')
             time.sleep(60)
-            op_data(days=3650)  # 数据清理和聚合
+            op_data(days=100)  # 数据清理和聚合
 
         t.sleep_minutes = 30  # 同步前休眠时间
         t.tb_file()

mdbq/mysql/mysql.py

@@ -57,7 +57,7 @@ class MysqlUpload:
         }
         self.filename = None
 
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], icm_up=[], df_sql=False, drop_dup=False, drop_duplicates=False, filename=None, count=None, json_path=None, system_name=None):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], icm_up=[], service_databases=[{'home_lx': 'mysql'}], df_sql=False, drop_dup=False, drop_duplicates=False, filename=None, count=None, json_path=None, system_name=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -167,11 +167,11 @@ class MysqlUpload:
             # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
             # ⚠️ mysql_all_dtypes 函数默认只读取 home_lx 的数据库信息，不会读取其他系统
             if cl and db_n and tb_n:
-                mysql_types.mysql_all_dtypes(db_name=db_name, table_name=table_name, system_name=system_name)  # 更新一个表的 dtypes
+                mysql_types.mysql_all_dtypes(service_databases=service_databases, db_name=db_name, table_name=table_name, system_name=system_name)  # 更新一个表的 dtypes
             elif cl and db_n:
-                mysql_types.mysql_all_dtypes(db_name=db_name, system_name=system_name)  # 更新一个数据库的 dtypes
+                mysql_types.mysql_all_dtypes(service_databases=service_databases, db_name=db_name, system_name=system_name)  # 更新一个数据库的 dtypes
             elif cl:
-                mysql_types.mysql_all_dtypes(system_name=system_name)  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
+                mysql_types.mysql_all_dtypes(service_databases=service_databases, system_name=system_name)  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
 
             # 4. 移除指定日期范围内的数据，仅限于聚合数据使用，其他情况不要设置
             if drop_duplicates and '日期' in df.columns.tolist():

{mdbq-1.4.1.dist-info → mdbq-1.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.4.1
+Version: 1.4.3
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.4.1.dist-info → mdbq-1.4.3.dist-info}/RECORD

@@ -3,15 +3,15 @@ mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/aggregation.py,sha256=cEKhmlqvrChXlxyxEq5qNMAgL_3hp1xtt9heOxfu9Z4,58113
 mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
-mdbq/aggregation/mysql_types.py,sha256=jTQaCrDy9hj36GIImshXANyQNFAvVKJTybkzJSh-qJ8,10671
+mdbq/aggregation/mysql_types.py,sha256=3j_SUTi7qYJyY3JtV_lRLIyczIczmRPE_WaIrXiZ6Rw,11425
 mdbq/aggregation/optimize_data.py,sha256=MnGEO4csKYHJ8MzzfsOf-r7yQbjO4ugthwdI-g6PRr4,4144
-mdbq/aggregation/query_data.py,sha256=qSCN3Y20Ken4cmwG63kMnvz9jg115PB9qvo-hF9Aq34,27504
+mdbq/aggregation/query_data.py,sha256=2SFeTkgsNgQVjNVDi1K2zTsNSw9dwL19Xj7Peu7sfRg,29828
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=TnG0t6Ted9miMoBdGanpTmj6AUa0YFSyEBvmgN3V7sw,87558
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=NJWEnxOZhbnqbHPQfCxBNAcl-LLLEk4Gn9QP35XlA0c,17041
+mdbq/company/copysh.py,sha256=XBnSalNHJPCBkswyiDpHhhVPqxRMOGcbAncWytrhCyQ,17039
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=bp6bVARZVm3ANj1pmM9hLB8Ao539TUWeM9xxeSsBpzw,5994
 mdbq/config/products.py,sha256=9gqXJMsw8KKuD4Xs6krNgcF7AuWDvV7clI6wVi3QjcA,4260
@@ -24,7 +24,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=kkkbFJHox0diPpDadwHicRQcDUxdH1h8MsMF1MNNBRI,44344
+mdbq/mysql/mysql.py,sha256=PXgE5mjLzzFDzor7DmcudEYz0pwHZRiiFomdkBmGj7U,44497
 mdbq/mysql/s_query.py,sha256=a33aYhW6gAnspIZfQ7l23ePln9-MD1f_ukypr5M0jd8,8018
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -35,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.4.1.dist-info/METADATA,sha256=6CQIPEW8CigYNwESE70LBIjNpEAGVSQ1wR1HRkHA8Kk,245
-mdbq-1.4.1.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.4.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.4.1.dist-info/RECORD,,
+mdbq-1.4.3.dist-info/METADATA,sha256=O4RKaUpul7RzqaucQbThigiFnKGCDN9K8TVsiZKZvng,245
+mdbq-1.4.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-1.4.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.4.3.dist-info/RECORD,,

{mdbq-1.4.1.dist-info → mdbq-1.4.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any